From c4e746b314173f3cac6f4c22d8a20524baea4559 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 21:20:05 -0700 Subject: [PATCH 1/5] feat(vmm): run virtio-net on vhost-net with configurable queue pairs --- dstack/vmm/rpc/proto/vmm_rpc.proto | 37 +- dstack/vmm/src/app.rs | 316 +++-- dstack/vmm/src/app/network.rs | 669 +++++++++- dstack/vmm/src/app/qemu.rs | 359 +++++- dstack/vmm/src/app/vm_info.rs | 201 ++- dstack/vmm/src/app/workdir.rs | 2 +- dstack/vmm/src/config.rs | 422 ++++++- dstack/vmm/src/main.rs | 16 + dstack/vmm/src/main_service.rs | 1103 ++++++++++++++++- dstack/vmm/src/netd.rs | 637 ++++++++-- dstack/vmm/src/one_shot.rs | 44 +- dstack/vmm/src/vmm-cli.py | 203 ++- .../vmm/ui/src/components/CreateVmDialog.ts | 72 +- .../vmm/ui/src/components/UpdateVmDialog.ts | 82 +- dstack/vmm/ui/src/composables/useVmManager.ts | 91 +- dstack/vmm/ui/src/styles/main.css | 26 +- dstack/vmm/ui/src/templates/app.html | 13 +- dstack/vmm/vmm.toml | 47 +- 18 files changed, 3938 insertions(+), 402 deletions(-) diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index c035e64d0..31fef51f4 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -39,19 +39,34 @@ message VmInfo { repeated GuestEvent events = 14; // Effective network interfaces resolved against node config. repeated NetworkInterfaceStatus interfaces = 15; + // Whether a QEMU process exists for this VM right now. The interfaces above + // are what it built only while this is true; otherwise they are what the next + // launch would build. + bool running = 16; } // Runtime status of a resolved VM network interface. message NetworkInterfaceStatus { - // Product-facing mode: "user", "bridge", or "custom". + // Product-facing mode: "user", "bridge", "macvtap", or "custom". string mode = 1; - // QEMU/network backend shape: "slirp", "tap_bridge", or "custom". + // QEMU/network backend shape: "slirp", "tap_bridge", "macvtap", or "custom". string backend = 2; string mac = 3; // Linux bridge name for tap_bridge backend. optional string bridge_name = 4; // QEMU netdev id, e.g. "net0". optional string netdev_id = 5; + // Effective vhost-net data plane state for this interface. Absent for custom + // mode, where the operator supplies the whole netdev string and the VMM does + // not parse it, so it knows of no data-plane state to report. + optional bool vhost = 6; + // Effective virtio-net queue pairs. Absent for custom mode, for the same + // reason as vhost. + optional uint32 queues = 7; + // Effective macvtap forwarding mode, for macvtap interfaces only. Node + // configuration decides it, so it is reported here with the rest of the + // resolved state rather than on the VM's own NetworkingConfig. + optional string macvtap_mode = 8; } // Structured log or lifecycle event emitted by the guest or runtime. @@ -132,9 +147,16 @@ message NetworkingConfig { string bridge_name = 2; // Parent host interface for macvtap mode. string parent = 3; - // Effective macvtap forwarding mode in responses. Deployment requests must - // leave this empty because the mode is controlled by node configuration. + // Deployment requests must leave this empty: the forwarding mode is node + // configuration, and a VM never pins one. The effective value is reported on + // NetworkInterfaceStatus.macvtap_mode instead. string macvtap_mode = 4; + // Move packet processing into the host kernel vhost-net data plane. Unset + // inherits the node default. User mode has no vhost backend and ignores it. + optional bool vhost = 5; + // virtio-net queue pairs. Unset inherits the node default. Bounded by the + // node's cvm.max_net_queues. + optional uint32 queues = 6; } // Requested GPU layout for a CVM. @@ -301,6 +323,13 @@ message NetworkingCapabilities { reserved "forward_service_enabled"; // Default bridge configured in vmm.toml [cvm.networking].bridge. string default_bridge = 4; + // Largest virtio-net queue pair count a deployment request may ask for. + uint32 max_queues = 5; + // Whether the node's own backend runs the vhost-net data plane. A NIC that + // pins neither vhost nor a queue count follows this, and without vhost the + // queue count does not scale with vCPUs -- so a client cannot describe what + // an empty queue field will do without it. + bool default_vhost = 6; } // Aggregated metadata exposed through GetMeta. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 836176d4b..e9205b07d 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -3,7 +3,10 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - config::{Config, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, Protocol}, + config::{ + Config, NetdInterface, Networking, NetworkingMode, NicNetworking, ProcessAnnotation, + Protocol, + }, logrotate, netd::{ self, InterfaceIdentity, PrepareBridgeRequest, PrepareMacvtapRequest, @@ -32,6 +35,7 @@ use rand::seq::SliceRandom; use serde::{Deserialize, Serialize}; use serde_json::json; use sha2::{Digest, Sha256}; +use std::cell::OnceCell; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use std::net::IpAddr; use std::path::{Path, PathBuf}; @@ -42,9 +46,15 @@ use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; pub(crate) use network::{ - resolve_networking, resolved_networks, validate_resolved_network, validate_resolved_networks, + clamp_queues_without_netd, filters_bridge_traffic, needs_netd_interface, netd_available, + netd_teardown, resolve_networking, resolved_networks, settle_vhost, validate_resolved_network, + validate_resolved_networks, }; pub use qemu::VmConfig; +// Exported so the RPC layer can assert that everything it reports is +// something it also accepts. +#[cfg(test)] +pub(crate) use vm_info::networking_to_proto; pub use workdir::VmWorkDir; mod host_share; @@ -126,7 +136,7 @@ pub struct Manifest { #[serde(default)] pub swtpm: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub networks: Vec, + pub networks: Vec, #[serde(default)] pub volumes: Vec, } @@ -355,7 +365,7 @@ impl App { let vm_id = manifest.id.clone(); let mut runtime_networks = vm_work_dir.runtime_networks(); if runtime_networks.is_empty() && cids_assigned.contains_key(&vm_id) { - runtime_networks = resolved_networks(&manifest, &self.config.cvm); + runtime_networks = self.inferred_runtime_networks(&manifest); if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); } @@ -454,7 +464,7 @@ impl App { append_boot_separator(&path); } - let mut runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); + let mut runtime_networks = self.runtime_networks(&vm_config.manifest); let devices = self.try_allocate_gpus(&vm_config.manifest)?; let gpu_host_config = self.config.cvm.gpu.clone(); let devices_to_sanitize = devices.clone(); @@ -547,19 +557,16 @@ impl App { vm: &VmConfig, networks: &mut [Networking], ) -> Result<()> { - if self.config.cvm.network_filter.mode == NetworkFilterMode::None - && !networks - .iter() - .any(|network| network.mode == NetworkingMode::Macvtap) + if !networks + .iter() + .any(|network| needs_netd_interface(network, &self.config.cvm)) { return Ok(()); } let qemu_uid = Uid::effective().as_raw(); let mut prepared = Vec::new(); for (nic_index, network) in networks.iter_mut().enumerate() { - if network.mode == NetworkingMode::Bridge - && self.config.cvm.network_filter.mode == NetworkFilterMode::None - { + if !needs_netd_interface(network, &self.config.cvm) { continue; } let identity = InterfaceIdentity { @@ -572,21 +579,28 @@ impl App { &network.mac_prefix_bytes(), nic_index, ); - let request = match network.mode { + let queues = network.queue_pairs(); + let filtered = filters_bridge_traffic(network, &self.config.cvm); + let request = match network.nic.mode { NetworkingMode::Bridge => NetdRequest::PrepareBridge(PrepareBridgeRequest { identity: identity.clone(), - bridge: network.bridge.clone(), + bridge: network.nic.bridge.clone(), mac, qemu_uid, - filter: self.config.cvm.network_filter.filter.clone(), - parameters: self.config.cvm.network_filter.parameters.clone(), + // Which filter, and with what parameters, is netd's to + // decide from its own configuration. An unfiltered TAP is + // only asked for by multiqueue, where the node may not run + // libvirt at all. + filtered, + queues, }), NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap(PrepareMacvtapRequest { identity: identity.clone(), - parent: network.parent.clone(), + parent: network.nic.parent.clone(), mac, qemu_uid, mode: network.macvtap_mode.clone(), + queues, }), NetworkingMode::User | NetworkingMode::Custom => continue, }; @@ -600,73 +614,224 @@ impl App { &self.config.netd.socket, &NetdRequest::Remove { identity: identity.clone(), + filtered, }, ) .await { warn!(%cleanup_error, "failed to roll back in-flight filtered network"); } - for identity in prepared.into_iter().rev() { - if let Err(cleanup_error) = netd::request( - &self.config.netd.socket, - &NetdRequest::Remove { identity }, - ) - .await - { - warn!(%cleanup_error, "failed to roll back prepared filtered network"); - } - } - return Err(error).context("failed to prepare libvirt-filtered networking"); + self.roll_back_prepared_networks(prepared).await; + // netd's own message is about a TAP, not about queues, so + // a caller who asked for multiqueue would not see their + // request named anywhere in the failure. + let unreachable = netd::is_unreachable(&error); + let error = Err(error).context("failed to prepare netd-managed networking"); + return if queues > 1 && unreachable { + error.with_context(|| { + format!( + "interface {nic_index} asked for {queues} queue pairs, which needs \ + a netd running on this host" + ) + }) + } else if queues > 1 { + error.with_context(|| { + format!("interface {nic_index} asked for {queues} queue pairs") + }) + } else { + error + }; } }; - if network.mode == NetworkingMode::Macvtap { - network.device = response - .device - .context("netd response omitted macvtap device")?; + prepared.push((identity.clone(), filtered)); + // netd built this one. Record it now, before anything else can + // fail, so teardown never has to re-derive it from a node + // configuration the operator may since have changed. + network.netd_interface = if filtered { + NetdInterface::Filtered + } else { + NetdInterface::Unfiltered + }; + // Everything below runs after netd already built a host interface, + // so a failure has to unwind the same way a failed Prepare does. + let accepted = (|| { + if network.nic.mode == NetworkingMode::Macvtap { + network.device = response + .device + .clone() + .context("netd response omitted macvtap device")?; + } + // QEMU refuses a TAP whose IFF_MULTI_QUEUE state disagrees with + // its own `queues=`, and reports it from inside the per-VM + // launcher. netd echoes what it built, so a netd too old to + // understand the request fails here, where the reason is + // legible. + if queues > 1 && response.queues != Some(queues) { + bail!( + "netd prepared interface {nic_index} with {} queue pairs instead of \ + {queues}; its version may predate multiqueue support", + response.queues.map_or_else( + || "an unreported number of".to_string(), + |q| q.to_string() + ) + ); + } + Ok(()) + })(); + if let Err(error) = accepted { + self.roll_back_prepared_networks(prepared).await; + return Err(error); } - prepared.push(identity); } Ok(()) } + /// The NICs a VM has now, or would get if it were started. + /// + /// While QEMU is up this is what the launch actually built. Once it is + /// down the snapshot describes a boot that is over: the node configuration + /// and the VM's own manifest can both have changed since, so reporting it + /// would answer a question about the past with the grammar of the present. + /// Predict instead, the same way the next launch will -- including the + /// drop to a single queue pair on a node with no netd. + /// + /// `netd_reachable` is shared across a request rather than probed here: + /// the probe is a blocking connect that netd's serialized accept loop has + /// to service, and one status query covers many VMs. + fn effective_networks( + &self, + info: &vm_info::VmInfo, + netd_reachable: &OnceCell, + ) -> Vec { + if info.running && !info.runtime_networks.is_empty() { + return info.runtime_networks.clone(); + } + let available = *netd_reachable.get_or_init(|| netd_available(&self.config.netd.socket)); + self.merge_networks(&info.manifest, available).0 + } + + /// Launch-time view of a VM's NICs: node defaults merged in, the + /// vCPU-scaled queue count made concrete, and multiqueue dropped when this + /// node has no netd to build the interface. + pub(crate) fn runtime_networks(&self, manifest: &Manifest) -> Vec { + let available = netd_available(&self.config.netd.socket); + let (networks, clamped, vhost_denied) = self.merge_networks(manifest, available); + if clamped > 0 { + warn!( + id = %manifest.id, + "netd is not available, so {clamped} bridge interface(s) fall back to a single \ + queue pair; run dstack-vmm netd to let queue pairs scale with vCPUs" + ); + } + if vhost_denied > 0 { + warn!( + id = %manifest.id, + "no qemu-bridge-helper found, so {vhost_denied} bridge interface(s) fall back to \ + the non-vhost bridge netdev; set cvm.qemu_bridge_helper to enable vhost" + ); + } + networks + } + + /// A running VM whose snapshot is missing, because a VMM that predates the + /// snapshot -- or predates it recording what netd built -- started it. + /// + /// Guessing is all that is left, so guess the way that VMM would have, and + /// then write the guess down. Leaving the marker unset would make every + /// later teardown re-derive it from node configuration that may by then + /// have moved, which is the failure this snapshot exists to prevent. + /// + /// The way *that* VMM would have, not this one: a build old enough to leave + /// no snapshot had no vhost and no multiqueue at all, so whatever this + /// node's defaults say now, the QEMU process actually running was given one + /// queue pair and no vhost. Asking `runtime_networks` would apply today's + /// defaults to a launch that predates them, and the guess is persisted, so + /// it would keep describing that VM wrongly for the life of its boot. + /// + /// `merge_networks` rather than `runtime_networks` for the same reason: the + /// latter probes netd and warns about a multiqueue fallback, which says + /// nothing about a VM that is already up. + fn inferred_runtime_networks(&self, manifest: &Manifest) -> Vec { + let mut networks = self.merge_networks(manifest, false).0; + for network in &mut networks { + network.nic.vhost = Some(false); + network.nic.queues = Some(1); + } + for network in &mut networks { + network.netd_interface = match netd_teardown(network, &self.config.cvm) { + Some(true) => NetdInterface::Filtered, + Some(false) => NetdInterface::Unfiltered, + None => NetdInterface::None, + }; + } + networks + } + + /// The merge itself, without the launch-time logging, plus how many NICs + /// lost multiqueue for want of netd. + fn merge_networks( + &self, + manifest: &Manifest, + netd_reachable: bool, + ) -> (Vec, usize, usize) { + let requested = if manifest.networks.is_empty() { + vec![self.config.cvm.networking.nic.clone()] + } else { + manifest.networks.clone() + }; + let mut resolved = resolved_networks(manifest, &self.config.cvm); + let clamped = + clamp_queues_without_netd(&requested, &mut resolved, &self.config.cvm, netd_reachable); + let vhost_denied = settle_vhost(&mut resolved, &self.config.cvm); + (resolved, clamped, vhost_denied) + } + + /// Removes interfaces netd already built for a launch that then failed. + async fn roll_back_prepared_networks(&self, prepared: Vec<(InterfaceIdentity, bool)>) { + for (identity, filtered) in prepared.into_iter().rev() { + if let Err(cleanup_error) = netd::request( + &self.config.netd.socket, + &NetdRequest::Remove { identity, filtered }, + ) + .await + { + warn!(%cleanup_error, "failed to roll back prepared network interface"); + } + } + } + pub(crate) async fn remove_filtered_networks( &self, vm_id: &str, networks: &[Networking], ) -> Result<()> { - if self.config.cvm.network_filter.mode == NetworkFilterMode::None - && !networks - .iter() - .any(|network| network.mode == NetworkingMode::Macvtap) + if networks + .iter() + .all(|network| netd_teardown(network, &self.config.cvm).is_none()) { return Ok(()); } let mut first_error = None; for (nic_index, network) in networks.iter().enumerate().rev() { - if network.mode == NetworkingMode::Bridge - && self.config.cvm.network_filter.mode == NetworkFilterMode::None - { - continue; - } - if !matches!( - network.mode, - NetworkingMode::Bridge | NetworkingMode::Macvtap - ) { + let Some(filtered) = netd_teardown(network, &self.config.cvm) else { continue; - } + }; let identity = InterfaceIdentity { instance_id: self.config.cvm.instance_id.clone(), vm_id: vm_id.to_string(), nic_index, }; - if let Err(error) = - netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity }).await + if let Err(error) = netd::request( + &self.config.netd.socket, + &NetdRequest::Remove { identity, filtered }, + ) + .await { first_error.get_or_insert(error); } } if let Some(error) = first_error { - return Err(error).context("failed to remove libvirt-filtered networking"); + return Err(error).context("failed to remove netd-managed networking"); } Ok(()) } @@ -1059,7 +1224,7 @@ impl App { let already_running = cids_assigned.contains_key(&vm_id); let mut runtime_networks = vm_work_dir.runtime_networks(); if runtime_networks.is_empty() && already_running { - runtime_networks = resolved_networks(&manifest, &self.config.cvm); + runtime_networks = self.inferred_runtime_networks(&manifest); if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); } @@ -1160,11 +1325,15 @@ impl App { }); let total = infos.len() as u32; + // One probe for the whole page, and none at all when every VM is + // running and has its own snapshot to report. + let netd_reachable = OnceCell::new(); let vms = paginate(infos, request.page, request.page_size) .map(|vm| { let work_dir = self.work_dir(&vm.config.manifest.id)?; let info = vm.merged_info(vms.get(&vm.config.manifest.id), &work_dir); - Ok(info.to_pb(&self.config.gateway, &self.config.cvm, request.brief)) + let networks = self.effective_networks(&info, &netd_reachable); + Ok(info.to_pb(&self.config.gateway, request.brief, &networks)) }) .collect::>>()?; Ok(StatusResponse { @@ -1188,14 +1357,19 @@ impl App { pub async fn vm_info(&self, id: &str) -> Result> { let proc_state = self.supervisor.info(id).await?; - let state = self.lock(); - let Some(vm_state) = state.get(id) else { - return Ok(None); + // Snapshot under the lock, then release it: describing the VM can + // probe netd, and that is a blocking connect the global state lock has + // no business being held across. + let info = { + let state = self.lock(); + let Some(vm_state) = state.get(id) else { + return Ok(None); + }; + vm_state.merged_info(proc_state.as_ref(), &self.work_dir(id)?) }; - let info = vm_state - .merged_info(proc_state.as_ref(), &self.work_dir(id)?) - .to_pb(&self.config.gateway, &self.config.cvm, false); - Ok(Some(info)) + let netd_reachable = OnceCell::new(); + let networks = self.effective_networks(&info, &netd_reachable); + Ok(Some(info.to_pb(&self.config.gateway, false, &networks))) } pub(crate) fn vm_event_report(&self, cid: u32, event: &str, body: String) -> Result<()> { @@ -1917,7 +2091,7 @@ mod tests { } use crate::config::{ - load_config_figment, CvmPlatform, Networking, NetworkingMode, TdxAttestationVariantConfig, + load_config_figment, CvmPlatform, NetworkingMode, TdxAttestationVariantConfig, }; use dstack_types::{ TdxImageMeasurement, TdxMrtdCandidates, TdxOsImageMeasurement, @@ -2230,17 +2404,10 @@ mod tests { )); let workdir = VmWorkDir::new(&temp); let mut manifest = test_manifest(1024); - manifest.networks = vec![Networking { + manifest.networks = vec![NicNetworking { mode: NetworkingMode::Bridge, bridge: "dstack-br0".to_string(), - parent: String::new(), - macvtap_mode: String::new(), - device: String::new(), - mac_prefix: String::new(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, - netdev: String::new(), + ..NicNetworking::default() }]; workdir.put_manifest(&manifest)?; @@ -2495,17 +2662,10 @@ mod tests { fn vm_measurement_config_ignores_networking_changes() -> Result<()> { let config = test_tdx_config()?; let mut bridge_manifest = test_manifest(2048); - bridge_manifest.networks = vec![Networking { + bridge_manifest.networks = vec![NicNetworking { mode: NetworkingMode::Bridge, bridge: "dstack-br0".to_string(), - parent: String::new(), - macvtap_mode: String::new(), - device: String::new(), - mac_prefix: "02:aa:bb".to_string(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, - netdev: String::new(), + ..NicNetworking::default() }]; let user_manifest = test_manifest(2048); let image = test_tdx_image(true); diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index 1b840154f..4017e89ff 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -10,57 +10,267 @@ use anyhow::{bail, Result}; use sha2::{Digest, Sha256}; use super::Manifest; -use crate::config::{CvmConfig, Networking, NetworkingMode}; +use crate::config::{ + CvmConfig, NetdInterface, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, + MAX_NET_QUEUES, +}; -pub(crate) fn resolve_networking(networking: &Networking, cfg: &CvmConfig) -> Networking { +/// Node configuration merged with what one NIC pins. +/// +/// The node half is taken wholesale and the NIC half overrides it where it is +/// set. Nothing the node owns -- the macvtap forwarding mode, the MAC prefix, +/// the user-mode network parameters, a custom netdev string -- can be +/// overridden here any more, because a [`NicNetworking`] cannot carry one. +pub(crate) fn resolve_networking( + networking: &NicNetworking, + cfg: &CvmConfig, + vcpu: u32, +) -> Networking { let mut resolved = cfg.networking.clone(); - resolved.mode = networking.mode; - resolved.restrict = cfg.networking.restrict || networking.restrict; + // A deployment that only tuned the data plane never named a backend, so + // the node keeps deciding which one this NIC uses -- including after the + // operator changes it. + resolved.nic.inherit_mode = networking.inherit_mode; + resolved.nic.mode = if networking.inherit_mode { + cfg.networking.nic.mode + } else { + networking.mode + }; + // Runtime state, never inherited from configuration or from a previous + // launch. Interface preparation sets both for the NICs it builds, and a + // node configuration that names either is rejected at startup. + resolved.netd_interface = crate::config::NetdInterface::None; + resolved.device.clear(); if !networking.bridge.is_empty() { - resolved.bridge = networking.bridge.clone(); + resolved.nic.bridge = networking.bridge.clone(); } if !networking.parent.is_empty() { - resolved.parent = networking.parent.clone(); - } - if !networking.mac_prefix.is_empty() { - resolved.mac_prefix = networking.mac_prefix.clone(); - } - if !networking.net.is_empty() { - resolved.net = networking.net.clone(); + resolved.nic.parent = networking.parent.clone(); } - if !networking.dhcp_start.is_empty() { - resolved.dhcp_start = networking.dhcp_start.clone(); - } - if !networking.netdev.is_empty() { - resolved.netdev = networking.netdev.clone(); + if networking.vhost.is_some() { + resolved.nic.vhost = networking.vhost; } + // Make the vCPU-scaled default concrete here, so every later stage -- + // netd preparation, the QEMU arguments, and removal after a VMM restart -- + // reads one number instead of recomputing it from a vCPU count it may no + // longer have. + resolved.nic.queues = Some(match networking.queues { + // An explicit count is honoured whatever the data plane: multiqueue + // without vhost is a valid, if unusual, thing to ask for. + Some(queues) => queues, + // Without vhost the QEMU main loop drains every queue on one thread, + // so scaling up buys almost nothing while still costing a netd + // interface, extra vectors, and a changed device. Anyone turning vhost + // off is asking for the old data plane; give them the old shape too. + None if resolved.vhost_enabled() => { + Networking::default_queue_pairs(vcpu, cfg.max_net_queues) + } + None => 1, + }); resolved } pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec { - if manifest.networks.is_empty() { - vec![cfg.networking.clone()] + let node_default = [cfg.networking.nic.clone()]; + let requested = if manifest.networks.is_empty() { + &node_default[..] } else { - manifest - .networks - .iter() - .map(|networking| resolve_networking(networking, cfg)) - .collect() + &manifest.networks[..] + }; + requested + .iter() + .map(|networking| resolve_networking(networking, cfg, manifest.vcpu)) + .collect() +} + +/// Whether netd must pre-create the host interface for this NIC. +/// +/// Macvtap always needs one. A bridge NIC needs one when libvirt filtering +/// binds an nwfilter to the TAP, and when multiqueue requires a persistent +/// `IFF_MULTI_QUEUE` device that `qemu-bridge-helper` cannot create. +pub(crate) fn needs_netd_interface(networking: &Networking, cfg: &CvmConfig) -> bool { + match networking.nic.mode { + NetworkingMode::Macvtap => true, + NetworkingMode::Bridge => { + cfg.network_filter.mode == NetworkFilterMode::Libvirt || networking.queue_pairs() > 1 + } + NetworkingMode::User | NetworkingMode::Custom => false, + } +} + +/// Whether this NIC's host interface carries a libvirt nwfilter binding. +/// Macvtap never does, and a bridge NIC only does when the node filters. +pub(crate) fn filters_bridge_traffic(networking: &Networking, cfg: &CvmConfig) -> bool { + networking.nic.mode == NetworkingMode::Bridge + && cfg.network_filter.mode == NetworkFilterMode::Libvirt +} + +/// Whether netd built this NIC's host interface, and if so whether it carries +/// an nwfilter binding. +/// +/// Interface preparation records this, because it is not derivable afterwards: +/// an operator can change `network_filter.mode` or `max_net_queues` while a VM +/// runs, and teardown has to undo what was built rather than what would be +/// built now. +pub(crate) fn netd_teardown(networking: &Networking, cfg: &CvmConfig) -> Option { + match networking.netd_interface { + NetdInterface::Filtered => Some(true), + NetdInterface::Unfiltered => Some(false), + // Either nothing was built, or this entry was persisted before + // preparation recorded the fact. Fall back to the derivation such an + // entry was created by; a Remove for an interface that does not exist + // is a no-op. + NetdInterface::None if needs_netd_interface(networking, cfg) => { + Some(filters_bridge_traffic(networking, cfg)) + } + NetdInterface::None => None, } } +/// Drops a NIC back to one queue pair when multiqueue would need a netd +/// interface this node cannot provide. +/// +/// Queue pairs are a default now, not something the operator asked for, so a +/// node that has never deployed netd must keep launching bridge VMs. An +/// explicit per-VM request is left alone: the caller asked for it, and failing +/// at prepare tells them why far better than silently halving their throughput. +/// Returns how many NICs it dropped, so a launch can say so and a status +/// query, which runs the same calculation to describe a stopped VM, stays +/// silent. +pub(crate) fn clamp_queues_without_netd( + requested: &[NicNetworking], + resolved: &mut [Networking], + cfg: &CvmConfig, + netd_available: bool, +) -> usize { + if netd_available { + return 0; + } + let mut clamped = 0; + for (networking, asked) in resolved.iter_mut().zip(requested) { + // Macvtap has nothing to fall back to: netd is the only thing that can + // create the device, so clamping one would describe a VM that cannot + // start either way. + if networking.nic.mode != NetworkingMode::Bridge + || asked.queues.is_some() + || !needs_netd_interface(networking, cfg) + // Filtering needs netd whatever the queue count, so dropping this + // NIC to one queue pair would not make it launchable. It would only + // describe it as something no launch can produce, and warn about a + // fallback that is not happening. + || filters_bridge_traffic(networking, cfg) + { + continue; + } + networking.nic.queues = Some(1); + clamped += 1; + } + clamped +} + +/// Locations distributions install `qemu-bridge-helper` in. The helper is +/// setuid root and attaches an unprivileged TAP to a whitelisted bridge, which +/// is how bridge mode avoids giving the VMM `CAP_NET_ADMIN`. +const BRIDGE_HELPER_CANDIDATES: [&str; 3] = [ + "/usr/lib/qemu/qemu-bridge-helper", + "/usr/libexec/qemu-bridge-helper", + "/usr/local/libexec/qemu-bridge-helper", +]; + +/// Absolute path of `qemu-bridge-helper`, which QEMU's `tap` netdev, unlike its +/// `bridge` netdev, has no compiled-in default for. +/// +/// A configured path is passed through unchecked: the operator is naming a +/// binary for QEMU to exec, and QEMU need not see this filesystem. +pub(crate) fn find_bridge_helper<'a>( + configured: &'a str, + candidates: &[&'a str], +) -> Option<&'a str> { + let configured = configured.trim(); + if !configured.is_empty() { + return Some(configured); + } + candidates + .iter() + .copied() + .find(|candidate| Path::new(candidate).exists()) +} + +pub(crate) fn bridge_helper(cfg: &CvmConfig) -> Option<&str> { + find_bridge_helper(&cfg.qemu_bridge_helper, &BRIDGE_HELPER_CANDIDATES) +} + +/// Whether this NIC will actually run on the vhost-net data plane. +/// +/// A bridge NIC that neither needs a netd interface nor can find +/// `qemu-bridge-helper` falls back to QEMU's `bridge` netdev, which has no +/// vhost support. Both the QEMU arguments and the reported status read this, +/// so a VM is never described as using a data plane it did not get. +pub(crate) fn effective_vhost(networking: &Networking, cfg: &CvmConfig) -> bool { + if !networking.vhost_enabled() { + return false; + } + networking.nic.mode != NetworkingMode::Bridge + || needs_netd_interface(networking, cfg) + || bridge_helper(cfg).is_some() +} + +/// Makes the effective data plane concrete on a launch-time NIC list, and +/// returns how many interfaces asked for vhost and did not get it. +/// +/// `vhost` on a freshly resolved entry is still a *request*: `None` means +/// inherit, and a bridge NIC that cannot reach `qemu-bridge-helper` runs on the +/// non-vhost netdev whatever it asked for. Settling it once, here, is what lets +/// the QEMU arguments and the reported status read the same value -- and keeps +/// them reading it after the operator moves the helper out from under a VM that +/// is already running. +pub(crate) fn settle_vhost(networks: &mut [Networking], cfg: &CvmConfig) -> usize { + let mut denied = 0; + for networking in networks.iter_mut() { + let effective = effective_vhost(networking, cfg); + if networking.vhost_enabled() && !effective { + denied += 1; + } + networking.nic.vhost = Some(effective); + } + denied +} + +/// Whether netd is reachable. A netd that died leaves its socket behind, so +/// existence alone would report a node as capable and fail every launch. +/// +/// A connect and nothing more, deliberately: netd serves connections serially, +/// so anything that waits for an answer reads a *busy* netd as a missing one +/// and silently drops the VM to a single queue pair. Accepting the connection +/// is the one signal that does not depend on what netd is doing right now. +pub(crate) fn netd_available(socket: &Path) -> bool { + std::os::unix::net::UnixStream::connect(socket).is_ok() +} + pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { - if networking.mode != NetworkingMode::Bridge { + // The vCPU-scaled default is bounded by construction; only an explicit + // request can exceed the hard cap. + if networking + .nic + .queues + .is_some_and(|queues| queues > MAX_NET_QUEUES) + { + bail!("networking queues must not exceed {MAX_NET_QUEUES}"); + } + if networking.nic.mode != NetworkingMode::Bridge { return Ok(()); } - if networking.bridge.is_empty() { + if networking.nic.bridge.is_empty() { bail!("bridge networking requested but no bridge is configured"); } if !Path::new("/sys/class/net") - .join(&networking.bridge) + .join(&networking.nic.bridge) .exists() { - bail!("bridge interface '{}' does not exist", networking.bridge); + bail!( + "bridge interface '{}' does not exist", + networking.nic.bridge + ); } Ok(()) } @@ -72,6 +282,53 @@ pub(crate) fn validate_resolved_networks(networks: &[Networking]) -> Result<()> Ok(()) } +/// Warns when a vhost NIC is about to launch on a host that has no +/// `/dev/vhost-net` at all. +/// +/// QEMU exits when `vhost=on` cannot open the device, and it does so from +/// inside the per-VM launcher where the reason is easy to miss. This puts the +/// remediation in the VMM log instead. +/// +/// Both checks are warnings, never refusals. QEMU is not necessarily this +/// process — an externally started supervisor can run it under another account +/// — so neither answers the question that decides the launch. They are the two +/// cheap statements that catch the two ways this actually goes wrong. +/// +/// The permission check matters because the node's mode is not uniform. It is +/// `root:kvm 0660` on Debian-family hosts, where a VMM in the `kvm` group is +/// fine, and `root:root 0600` on several others — where every bridge VM stops +/// restarting after an upgrade turns vhost on node-wide, with the only +/// explanation buried in a per-VM launcher's QEMU output. Existence alone says +/// nothing about that case, which is the likelier of the two. +pub(crate) fn warn_if_vhost_net_missing(networks: &[Networking]) { + const VHOST_NET: &str = "/dev/vhost-net"; + if !networks.iter().any(Networking::vhost_enabled) { + return; + } + // The node is a kmod static device node, so it is present even before + // vhost_net is loaded; QEMU's open autoloads the module. + if !Path::new(VHOST_NET).exists() { + tracing::warn!( + "{VHOST_NET} is missing; vhost networking will fail to start. load the vhost_net \ + module, or set vhost = false in [cvm.networking]" + ); + return; + } + if let Err(error) = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(VHOST_NET) + { + if error.kind() == std::io::ErrorKind::PermissionDenied { + tracing::warn!( + "{VHOST_NET} is not accessible to this process; if QEMU runs under the same \ + account, vhost networking will fail to start. add that account to the device's \ + group, or set vhost = false in [cvm.networking]" + ); + } + } +} + /// Derives a deterministic, locally administered unicast MAC address. /// /// Index zero preserves the legacy single-NIC derivation. Later interfaces @@ -98,7 +355,361 @@ pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) #[cfg(test)] mod tests { - use super::mac_address_for_vm_index; + use super::{ + clamp_queues_without_netd, effective_vhost, mac_address_for_vm_index, needs_netd_interface, + netd_teardown, resolve_networking, resolved_networks, settle_vhost, + validate_resolved_networks, + }; + use crate::config::{Networking, NetworkingMode, NicNetworking}; + + fn macvtap_network() -> NicNetworking { + NicNetworking { + mode: NetworkingMode::Macvtap, + parent: "eth0".into(), + // Most tests here exercise the vhost data plane, which the + // shipped default leaves off; opt in the way a real node would. + vhost: Some(true), + ..NicNetworking::default() + } + } + + /// The same NIC as a resolved value, for the checks that run against what + /// a launch would see rather than against what a VM pins. + fn macvtap_resolved() -> Networking { + Networking { + nic: macvtap_network(), + ..Networking::default() + } + } + + fn node_config(mode: NetworkingMode) -> crate::config::CvmConfig { + use rocket::figment::providers::Format as _; + let config: crate::config::Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + let mut cvm = config.cvm; + cvm.networking.nic.mode = mode; + cvm.networking.nic.bridge = "br0".into(); + cvm.networking.nic.parent = "eth0".into(); + // The shipped default leaves vhost off; the tests here are about the + // vhost data plane, so this node opts in the way a real one would. + cvm.networking.nic.vhost = Some(true); + cvm + } + + /// A node whose configuration never names `vhost` — the upgrade case, + /// where the toml predates the key entirely. + fn unconfigured_vhost_node(mode: NetworkingMode) -> crate::config::CvmConfig { + let mut cvm = node_config(mode); + cvm.networking.nic.vhost = None; + cvm + } + + fn manifest_with(vcpu: u32, networks: Vec) -> crate::app::Manifest { + let mut manifest: crate::app::Manifest = serde_json::from_value(serde_json::json!({ + "id": "vm-1", "name": "n", "app_id": "a", "vcpu": vcpu, "memory": 2048, + "disk_size": 10, "image": "i", "port_map": [], "created_at_ms": 0, + })) + .unwrap(); + manifest.networks = networks; + manifest + } + + #[test] + fn queue_pairs_default_to_the_vcpu_count_up_to_the_cap() { + let cvm = node_config(NetworkingMode::Bridge); + for (vcpu, want) in [(1, 1), (2, 2), (8, 8), (16, 16), (32, 16), (128, 16)] { + let resolved = resolved_networks(&manifest_with(vcpu, vec![]), &cvm); + assert_eq!( + resolved[0].queue_pairs(), + want, + "vcpu {vcpu} should give {want} queue pairs" + ); + } + } + + /// An upgraded node must keep building the device its VMs have always + /// had: userspace virtio, one queue pair. Both the toml that predates the + /// `vhost` key and the shipped default say so. + #[test] + fn a_node_that_never_asked_for_vhost_keeps_the_old_device_shape() { + let shipped = { + use rocket::figment::providers::Format as _; + let config: crate::config::Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + config.cvm.networking.nic.vhost + }; + for vhost in [None, shipped] { + let mut cvm = unconfigured_vhost_node(NetworkingMode::Bridge); + cvm.networking.nic.vhost = vhost; + let resolved = resolved_networks(&manifest_with(16, vec![]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + assert!(!needs_netd_interface(&resolved[0], &cvm)); + } + } + + #[test] + fn turning_vhost_off_also_turns_off_the_multiqueue_default() { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.networking.nic.vhost = Some(false); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + assert!(!needs_netd_interface(&resolved[0], &cvm)); + + // Per-VM opt-out does the same thing. + let cvm = node_config(NetworkingMode::Bridge); + let mut asked = cvm.networking.nic.clone(); + asked.vhost = Some(false); + let resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 1); + + // But an explicit queue count is still honoured without vhost. + let mut asked = cvm.networking.nic.clone(); + asked.vhost = Some(false); + asked.queues = Some(4); + let resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 4); + } + + #[test] + fn lowering_the_request_ceiling_also_lowers_the_default() { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.max_net_queues = 2; + let resolved = resolved_networks(&manifest_with(16, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 2); + + // Raising it past the scaling cap widens requests, not the default. + cvm.max_net_queues = 32; + let resolved = resolved_networks(&manifest_with(24, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 16); + } + + #[test] + fn status_never_claims_a_data_plane_the_nic_did_not_get() { + let mut cvm = node_config(NetworkingMode::Bridge); + // No helper on this filesystem and no netd interface needed, so the + // NIC falls back to QEMU's `bridge` netdev, which has no vhost. + cvm.qemu_bridge_helper = String::new(); + let mut single = cvm.networking.nic.clone(); + single.queues = Some(1); + let resolved = resolved_networks(&manifest_with(8, vec![single]), &cvm); + assert!(resolved[0].vhost_enabled()); + let fell_back = !effective_vhost(&resolved[0], &cvm); + assert_eq!(fell_back, super::bridge_helper(&cvm).is_none()); + + // A configured helper is taken at its word, so vhost is real. + cvm.qemu_bridge_helper = "/opt/qemu-bridge-helper".into(); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(effective_vhost(&resolved[0], &cvm)); + + // Multiqueue goes through netd, which needs no helper at all. + let mut mq = cvm.networking.nic.clone(); + mq.queues = Some(4); + cvm.qemu_bridge_helper = String::new(); + let resolved = resolved_networks(&manifest_with(8, vec![mq]), &cvm); + assert!(needs_netd_interface(&resolved[0], &cvm)); + assert!(effective_vhost(&resolved[0], &cvm)); + } + + #[test] + fn an_explicit_queue_count_survives_resolution() { + let cvm = node_config(NetworkingMode::Bridge); + let mut asked = macvtap_network(); + asked.mode = NetworkingMode::Bridge; + asked.queues = Some(2); + let resolved = resolved_networks(&manifest_with(16, vec![asked]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 2); + } + + #[test] + fn user_mode_stays_single_queue_whatever_the_vcpu_count() { + let cvm = node_config(NetworkingMode::User); + let resolved = resolved_networks(&manifest_with(32, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 1); + } + + #[test] + fn without_netd_a_defaulted_bridge_drops_to_one_queue_but_a_request_does_not() { + let cvm = node_config(NetworkingMode::Bridge); + + // The default is ours to lower: a node that never deployed netd must + // keep launching bridge VMs. + let requested = vec![cvm.networking.nic.clone()]; + let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 8); + clamp_queues_without_netd(&requested, &mut resolved, &cvm, false); + assert_eq!(resolved[0].queue_pairs(), 1); + assert!(!needs_netd_interface(&resolved[0], &cvm)); + + // An explicit request is left alone, so prepare fails where the caller + // can see why instead of silently halving their throughput. + let mut asked = cvm.networking.nic.clone(); + asked.queues = Some(4); + let requested = vec![asked.clone()]; + let mut resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); + clamp_queues_without_netd(&requested, &mut resolved, &cvm, false); + assert_eq!(resolved[0].queue_pairs(), 4); + + // With netd present nothing is touched. + let requested = vec![cvm.networking.nic.clone()]; + let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + clamp_queues_without_netd(&requested, &mut resolved, &cvm, true); + assert_eq!(resolved[0].queue_pairs(), 8); + } + + #[test] + fn validation_never_depends_on_this_process_reaching_vhost_net() { + // QEMU may run under different credentials, so a NIC that asks for + // vhost must validate on hosts where the VMM itself cannot open the + // device. Both of these hold whether or not /dev/vhost-net exists here. + let mut networking = macvtap_resolved(); + assert!(networking.vhost_enabled()); + validate_resolved_networks(&[networking.clone()]).unwrap(); + + networking.nic.queues = Some(4); + validate_resolved_networks(&[networking]).unwrap(); + } + + #[test] + fn queue_counts_above_the_hard_bound_are_rejected() { + let mut networking = macvtap_resolved(); + networking.nic.queues = Some(super::MAX_NET_QUEUES + 1); + let error = validate_resolved_networks(&[networking]).unwrap_err(); + assert!(error.to_string().contains("must not exceed")); + } + + /// The data plane a NIC actually gets is decided once, at launch, and + /// written into the runtime entry. Recomputing it later would let a report + /// about a running VM change under an operator's edit to node + /// configuration, describing a data plane QEMU is not using. + #[test] + fn settling_vhost_records_what_the_launch_decided() { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.qemu_bridge_helper = String::new(); + let mut single = cvm.networking.nic.clone(); + single.queues = Some(1); + let manifest = manifest_with(8, vec![single]); + + // Whether this host has a helper is not the test's business; that it + // gets written down, once, is. + let helper_missing = super::bridge_helper(&cvm).is_none(); + let mut networks = resolved_networks(&manifest, &cvm); + assert!(networks[0].vhost_enabled(), "the request starts out on"); + assert_eq!( + settle_vhost(&mut networks, &cvm), + usize::from(helper_missing) + ); + assert_eq!(networks[0].nic.vhost, Some(!helper_missing)); + // Settling an already-settled list reports nothing new, so a relaunch + // does not warn about a fallback that already happened. + assert_eq!(settle_vhost(&mut networks, &cvm), 0); + + // A configured helper is taken at its word, so the same NIC settles on. + cvm.qemu_bridge_helper = "/opt/qemu-bridge-helper".into(); + let mut with_helper = resolved_networks(&manifest, &cvm); + assert_eq!(settle_vhost(&mut with_helper, &cvm), 0); + assert_eq!(with_helper[0].nic.vhost, Some(true)); + + // The entry the first launch settled keeps its answer: nothing about a + // running VM is recomputed from the configuration as it stands now. + assert_eq!(networks[0].nic.vhost, Some(!helper_missing)); + } + + /// Dropping to a single queue pair is only worth doing when it makes the + /// NIC launchable. A filtered bridge needs netd whatever its queue count, + /// so clamping it would report a shape no launch can produce. + #[test] + fn a_filtered_bridge_is_not_clamped_because_it_cannot_help() { + use crate::config::NetworkFilterMode; + + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.network_filter.mode = NetworkFilterMode::Libvirt; + let requested = vec![cvm.networking.nic.clone()]; + let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 8); + assert_eq!( + clamp_queues_without_netd(&requested, &mut resolved, &cvm, false), + 0 + ); + assert_eq!(resolved[0].queue_pairs(), 8); + + // Unfiltered, the same NIC does drop, because then it can launch. + let cvm = node_config(NetworkingMode::Bridge); + let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert_eq!( + clamp_queues_without_netd(&requested, &mut resolved, &cvm, false), + 1 + ); + assert_eq!(resolved[0].queue_pairs(), 1); + } + + /// Teardown has to undo what was built. Node configuration is mutable and + /// a VM outlives an edit to it, so re-deriving "did netd build this?" at + /// removal time orphans TAPs and leaks nwfilter bindings whose ebtables + /// rules the next VM at the same deterministic interface name inherits. + #[test] + fn teardown_follows_what_was_built_not_what_configuration_now_says() { + use crate::config::{NetdInterface, NetworkFilterMode}; + + let filtering = { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.network_filter.mode = NetworkFilterMode::Libvirt; + cvm + }; + let unfiltered = node_config(NetworkingMode::Bridge); + + let mut built_filtered = filtering.networking.clone(); + built_filtered.nic.queues = Some(1); + built_filtered.netd_interface = NetdInterface::Filtered; + // The operator turns filtering off while the VM runs. The binding is + // still there and still has to be deleted. + assert_eq!(netd_teardown(&built_filtered, &unfiltered), Some(true)); + + let mut built_unfiltered = unfiltered.networking.clone(); + built_unfiltered.nic.queues = Some(4); + built_unfiltered.netd_interface = NetdInterface::Unfiltered; + // The operator turns filtering on. There is no binding to delete, and + // asking libvirt for one would fail the removal. + assert_eq!(netd_teardown(&built_unfiltered, &filtering), Some(false)); + + // A NIC netd never touched stays untouched, whatever the node now says. + let mut untouched = unfiltered.networking.clone(); + untouched.nic.queues = Some(1); + assert_eq!(netd_teardown(&untouched, &unfiltered), None); + + // An entry persisted before preparation recorded the fact still gets + // torn down by the rule that created it. + let mut legacy = filtering.networking.clone(); + legacy.nic.queues = Some(1); + assert_eq!(legacy.netd_interface, NetdInterface::None); + assert_eq!(netd_teardown(&legacy, &filtering), Some(true)); + } + + /// Resolution produces launch input, never a claim about what exists. + #[test] + fn resolution_never_carries_a_stale_interface_record() { + use crate::config::NetdInterface; + + let cvm = node_config(NetworkingMode::Bridge); + // Single queue and no filtering, so nothing but a stale record could + // make teardown believe netd built something. + let mut previous = cvm.networking.clone(); + previous.nic.queues = Some(1); + previous.netd_interface = NetdInterface::Filtered; + assert_eq!(netd_teardown(&previous, &cvm), Some(true)); + + let resolved = resolve_networking(&previous.nic, &cvm, 4); + assert_eq!(resolved.netd_interface, NetdInterface::None); + assert_eq!(netd_teardown(&resolved, &cvm), None); + } #[test] fn primary_mac_keeps_legacy_derivation_and_later_nics_are_distinct() { diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index db2fd39d4..94a6348fe 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -9,14 +9,15 @@ use super::{ hugepage_numa_nodes, image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, - network::{mac_address_for_vm_index, validate_resolved_networks}, + network::{ + bridge_helper, mac_address_for_vm_index, needs_netd_interface, validate_resolved_networks, + warn_if_vhost_net_missing, + }, pci_numa_node, round_up, GpuConfig, VmWorkDir, }; use crate::{ app::Manifest, - config::{ - CvmConfig, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, - }, + config::{CvmConfig, CvmPlatform, Networking, NetworkingMode, ProcessAnnotation}, netd::{tap_name, InterfaceIdentity}, vm_launcher::{ChildCommand, LaunchSpec, OpenFile}, }; @@ -167,6 +168,14 @@ fn create_hd( Ok(()) } +fn on_off(enabled: bool) -> &'static str { + if enabled { + "on" + } else { + "off" + } +} + fn virtio_pci_device(device: &str, snp: bool) -> String { if snp { format!("{device},disable-legacy=on,iommu_platform=true") @@ -179,6 +188,34 @@ struct PreparedVolume { source: String, } +/// First descriptor the per-VM launcher may hand to QEMU. Zero through two are +/// the standard streams. +const FIRST_INHERITED_FD: i32 = 3; + +/// Descriptors the launcher opens for each macvtap NIC, one per queue pair. +/// +/// Both the launcher's open list and the `-netdev` arguments derive from this +/// one layout, so they cannot disagree about which descriptor belongs to which +/// NIC. +fn macvtap_fd_layout(networks: &[Networking]) -> Vec> { + let mut next_fd = FIRST_INHERITED_FD; + networks + .iter() + .map(|network| { + if network.nic.mode != NetworkingMode::Macvtap { + return Vec::new(); + } + (0..network.queue_pairs()) + .map(|_| { + let fd = next_fd; + next_fd += 1; + fd + }) + .collect() + }) + .collect() +} + struct PreparedQemuLaunch { workdir: VmWorkDir, platform: CvmPlatform, @@ -209,6 +246,7 @@ impl PreparedQemuLaunch { let platform = cfg.resolved_platform(); let networks = networks.to_vec(); validate_resolved_networks(&networks)?; + warn_if_vhost_net_missing(&networks); let volumes = vm .manifest .volumes @@ -352,7 +390,7 @@ impl VmConfig { let has_macvtap = prepared .networks .iter() - .any(|network| network.mode == NetworkingMode::Macvtap); + .any(|network| network.nic.mode == NetworkingMode::Macvtap); let Some(socket) = prepared.swtpm_socket.as_deref() else { if has_macvtap { return self.wrap_launcher(&prepared, process, None, None); @@ -396,14 +434,17 @@ impl VmConfig { swtpm: Option, swtpm_socket: Option, ) -> Result> { + // Each queue pair is a separate open of the same macvtap character + // device; the kernel attaches one tap queue per open. let open_files = prepared .networks .iter() - .enumerate() - .filter(|(_, network)| network.mode == NetworkingMode::Macvtap) - .map(|(index, network)| OpenFile { - fd: (3 + index) as i32, - path: network.device.clone().into(), + .zip(macvtap_fd_layout(&prepared.networks)) + .flat_map(|(network, fds)| { + fds.into_iter().map(|fd| OpenFile { + fd, + path: network.device.clone().into(), + }) }) .collect(); let spec = LaunchSpec { @@ -589,11 +630,12 @@ impl QemuCommandBuilder<'_> { } fn configure_networking(&self, command: &mut Command) -> Result<()> { + let macvtap_fds = macvtap_fd_layout(&self.prepared.networks); let hostfwd_index = self .prepared .networks .iter() - .position(|networking| networking.mode == NetworkingMode::User); + .position(|networking| networking.nic.mode == NetworkingMode::User); for (index, networking) in self.prepared.networks.iter().enumerate() { let net_id = format!("net{index}"); let mac = mac_address_for_vm_index( @@ -601,12 +643,20 @@ impl QemuCommandBuilder<'_> { &networking.mac_prefix_bytes(), index, ); - let net_device = virtio_pci_device( - &format!("virtio-net-pci,netdev={net_id},mac={mac}"), - self.is_amd_sev_snp(), - ); - let netdev = match networking.mode { + let queues = networking.queue_pairs(); + let vhost = networking.vhost_enabled(); + let mut device = format!("virtio-net-pci,netdev={net_id},mac={mac}"); + if queues > 1 { + // One vector per queue direction, plus config and control. + device.push_str(&format!(",mq=on,vectors={}", 2 * queues + 2)); + } + let net_device = virtio_pci_device(&device, self.is_amd_sev_snp()); + let netdev = match networking.nic.mode { NetworkingMode::User => { + // The user-mode backend has neither, so both are ignored + // here. A caller who *named* this mode and then asked for + // vhost or more than one queue pair is refused by the RPC; + // one who inherited it is not, and lands here. let mut netdev = format!( "user,id={net_id},net={},dhcpstart={},restrict={}", networking.net, @@ -627,24 +677,46 @@ impl QemuCommandBuilder<'_> { netdev } NetworkingMode::Bridge => { - tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge); - match self.cfg.network_filter.mode { - NetworkFilterMode::None => { - format!("bridge,id={net_id},br={}", networking.bridge) - } - NetworkFilterMode::Libvirt => { - let tap = tap_name(&InterfaceIdentity { - instance_id: self.cfg.instance_id.clone(), - vm_id: self.vm.manifest.id.clone(), - nic_index: index, - }); - // Keep the filtered backend conservative: QEMU - // uses the TAP path on which libvirt installed the - // nwfilter binding instead of opening vhost-net. - format!( - "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost=off" - ) + tracing::info!( + "bridge networking: mac={mac} bridge={} vhost={vhost} queues={queues}", + networking.nic.bridge + ); + if needs_netd_interface(networking, self.cfg) { + // netd owns this TAP: libvirt filtering binds an + // nwfilter to it, and multiqueue needs the persistent + // IFF_MULTI_QUEUE device the bridge helper cannot make. + let tap = tap_name(&InterfaceIdentity { + instance_id: self.cfg.instance_id.clone(), + vm_id: self.vm.manifest.id.clone(), + nic_index: index, + }); + let mut netdev = format!( + "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost={}", + on_off(vhost) + ); + if queues > 1 { + netdev.push_str(&format!(",queues={queues}")); } + netdev + } else if let Some(helper) = vhost.then(|| bridge_helper(self.cfg)).flatten() { + // QEMU's `bridge` netdev has no vhost support, but the + // same setuid helper works behind a `tap` netdev, so + // the VMM still needs no network privileges. + format!( + "tap,id={net_id},br={},helper={helper},vhost=on", + networking.nic.bridge + ) + } else if vhost { + // vhost is a node-wide setting, so a node whose helper + // sits somewhere unusual must keep booting VMs rather + // than lose every bridge NIC to a path lookup. + tracing::warn!( + "{net_id}: no qemu-bridge-helper found, falling back to the \ + non-vhost bridge netdev. set cvm.qemu_bridge_helper to enable vhost" + ); + format!("bridge,id={net_id},br={}", networking.nic.bridge) + } else { + format!("bridge,id={net_id},br={}", networking.nic.bridge) } } NetworkingMode::Custom => { @@ -659,7 +731,23 @@ impl QemuCommandBuilder<'_> { if networking.device.is_empty() { bail!("macvtap interface {index} has not been prepared by netd"); } - format!("tap,id={net_id},fd={},vhost=off", 3 + index) + let fds = macvtap_fds + .get(index) + .filter(|fds| !fds.is_empty()) + .with_context(|| { + format!("macvtap interface {index} has no launcher descriptors") + })?; + let selector = if fds.len() == 1 { + format!("fd={}", fds[0]) + } else { + let fds = fds + .iter() + .map(|fd| fd.to_string()) + .collect::>() + .join(":"); + format!("fds={fds}") + }; + format!("tap,id={net_id},{selector},vhost={}", on_off(vhost)) } }; command.arg("-netdev").arg(netdev); @@ -1021,14 +1109,14 @@ mod tests { }; use super::{ - amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, - PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, + amd_sev_snp_memory_backend_arg, macvtap_fd_layout, parse_amd_sev_snp_qmp_capabilities, + virtio_pci_device, PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, }; use crate::app::image::{Image, ImageInfo}; use crate::app::{needs_swtpm, GpuConfig, GpuSpec, Manifest, PortMapping, VmVolume, VmWorkDir}; use crate::config::{ - Config, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, Protocol, - DEFAULT_CONFIG, + Config, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, + Protocol, DEFAULT_CONFIG, }; use crate::netd::{tap_name, InterfaceIdentity}; use dstack_types::{KeyProviderKind, TeeVariant}; @@ -1080,8 +1168,9 @@ mod tests { ); } - #[test] - fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { + /// Minimal launch fixture. Nothing it points at has to exist on disk; every + /// test overrides the fields it asserts on. + fn test_launch_fixture() -> (Config, VmConfig, PreparedQemuLaunch) { let mut config: Config = Figment::from(Toml::string(DEFAULT_CONFIG)) .extract() .unwrap(); @@ -1151,7 +1240,7 @@ mod tests { workdir: PathBuf::from("/does-not-exist/vm-1"), gateway_enabled: false, }; - let mut prepared = PreparedQemuLaunch { + let prepared = PreparedQemuLaunch { workdir: VmWorkDir::new("/does-not-exist/vm-1"), platform: CvmPlatform::Tdx, networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], @@ -1167,6 +1256,171 @@ mod tests { snp_host_data: None, snp_launch_params: None, }; + (config, vm, prepared) + } + + /// Builds the `-netdev`/`-device` pairs for one NIC layout. + fn net_args(config: &Config, networks: Vec) -> Vec { + let (_, vm, mut prepared) = test_launch_fixture(); + prepared.networks = networks; + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + process + .args + .windows(2) + .filter(|args| args[0] == "-netdev" || args[0] == "-device") + .map(|args| args[1].clone()) + .collect() + } + + fn bridge_network(config: &Config) -> Networking { + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::Bridge; + networking.nic.bridge = "br0".into(); + // The vhost tests below are about what an opted-in node builds; the + // shipped default leaves it off. + networking.nic.vhost = Some(true); + networking + } + + #[test] + fn bridge_vhost_uses_the_bridge_helper_behind_a_tap_netdev() { + // QEMU's `bridge` netdev has no vhost support at all, so enabling the + // kernel data plane has to switch netdev types while keeping the same + // unprivileged setuid helper. + let (mut config, ..) = test_launch_fixture(); + config.cvm.qemu_bridge_helper = "/usr/lib/qemu/qemu-bridge-helper".into(); + let args = net_args(&config, vec![bridge_network(&config)]); + assert!(args.contains( + &"tap,id=net0,br=br0,helper=/usr/lib/qemu/qemu-bridge-helper,vhost=on".to_string() + )); + // A single queue pair must keep the historical device line byte for byte. + assert!(args.iter().any( + |arg| arg.starts_with("virtio-net-pci,netdev=net0,mac=") && !arg.contains("mq=on") + )); + } + + #[test] + fn a_missing_bridge_helper_is_reported_rather_than_guessed() { + // Configured paths are trusted verbatim: QEMU execs them, and it need + // not share this filesystem. + assert_eq!( + crate::app::network::find_bridge_helper(" /opt/qemu-bridge-helper ", &[]), + Some("/opt/qemu-bridge-helper") + ); + assert_eq!( + crate::app::network::find_bridge_helper("", &["/nonexistent/a", "/nonexistent/b"]), + None + ); + } + + #[test] + fn disabling_vhost_restores_the_legacy_bridge_netdev() { + let (config, ..) = test_launch_fixture(); + let mut networking = bridge_network(&config); + networking.nic.vhost = Some(false); + let args = net_args(&config, vec![networking]); + assert!(args.contains(&"bridge,id=net0,br=br0".to_string())); + } + + #[test] + fn multiqueue_bridge_uses_the_netd_tap_and_derives_vectors() { + let (mut config, ..) = test_launch_fixture(); + config.cvm.instance_id = "vmm-a".into(); + let mut networking = bridge_network(&config); + networking.nic.queues = Some(4); + let args = net_args(&config, vec![networking]); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=on,queues=4" + ))); + // vectors = 2 per queue pair, plus config and control. + assert!(args.iter().any(|arg| arg.contains("mq=on,vectors=10"))); + } + + #[test] + fn macvtap_queues_take_one_inherited_descriptor_each() { + let (config, ..) = test_launch_fixture(); + let mut first = config.cvm.networking.clone(); + first.nic.mode = NetworkingMode::Macvtap; + first.nic.parent = "eth0".into(); + first.nic.vhost = Some(true); + first.device = "/dev/tap7".into(); + first.nic.queues = Some(2); + let mut second = first.clone(); + second.device = "/dev/tap9".into(); + second.nic.queues = Some(3); + + let networks = vec![first, second]; + let args = net_args(&config, networks.clone()); + assert!(args.contains(&"tap,id=net0,fds=3:4,vhost=on".to_string())); + assert!(args.contains(&"tap,id=net1,fds=5:6:7,vhost=on".to_string())); + + // The launcher must open exactly those descriptors, in that order. + let layout = macvtap_fd_layout(&networks); + assert_eq!(layout, vec![vec![3, 4], vec![5, 6, 7]]); + } + + #[test] + fn macvtap_keeps_a_single_fd_argument_for_one_queue() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::Macvtap; + networking.nic.parent = "eth0".into(); + networking.nic.vhost = Some(true); + networking.device = "/dev/tap7".into(); + let args = net_args(&config, vec![networking]); + assert!(args.contains(&"tap,id=net0,fd=3,vhost=on".to_string())); + } + + /// The operator owns a custom netdev string and the VMM cannot edit it, so + /// the generated device line must never claim more queues than that string + /// provides -- QEMU refuses the mismatch, from inside the per-VM launcher + /// where the reason is hard to see. + #[test] + fn custom_netdev_keeps_its_string_and_stays_single_queue() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::Custom; + networking.netdev = "tap,id=net0,ifname=custom0,vhost=on,queues=8".into(); + // Even a queue count that reached the entry some other way is ignored. + networking.nic.queues = Some(8); + let args = net_args(&config, vec![networking]); + assert!(args.contains(&"tap,id=net0,ifname=custom0,vhost=on,queues=8".to_string())); + assert!( + args.iter().all(|arg| !arg.contains("mq=on")), + "custom mode must not generate a multiqueue device line: {args:?}" + ); + } + + #[test] + fn user_mode_ignores_vhost_and_keeps_its_netdev() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::User; + networking.nic.vhost = Some(true); + let args = net_args(&config, vec![networking]); + assert!(args + .iter() + .any(|arg| arg.starts_with("user,id=net0,") && !arg.contains("vhost"))); + assert!(args.iter().any( + |arg| arg.starts_with("virtio-net-pci,netdev=net0,mac=") && !arg.contains("mq=on") + )); + } + + #[test] + fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { + let (mut config, vm, mut prepared) = test_launch_fixture(); let process = QemuCommandBuilder { vm: &vm, @@ -1232,8 +1486,9 @@ mod tests { .any(|arg| arg.contains("virtio-net-pci,netdev=net1"))); for network in &mut prepared.networks { - network.mode = NetworkingMode::Bridge; - network.bridge = "br0".into(); + network.nic.mode = NetworkingMode::Bridge; + network.nic.bridge = "br0".into(); + network.nic.vhost = Some(false); } let process = QemuCommandBuilder { vm: &vm, @@ -1266,6 +1521,10 @@ mod tests { assert!(process.args.iter().any(|arg| { arg == &format!("tap,id=net0,ifname={expected_tap},script=no,downscript=no,vhost=off") })); + assert!(process + .args + .iter() + .all(|arg| !arg.contains("mq=on") && !arg.contains("vectors="))); prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock")); let process = QemuCommandBuilder { @@ -1294,16 +1553,12 @@ mod tests { prepared.swtpm_socket = None; prepared.networks = vec![Networking { - mode: NetworkingMode::Custom, - bridge: String::new(), - parent: String::new(), - macvtap_mode: String::new(), - device: String::new(), - mac_prefix: String::new(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, + nic: NicNetworking { + mode: NetworkingMode::Custom, + ..NicNetworking::default() + }, netdev: "tap,id=wrong".into(), + ..Networking::default() }]; let error = QemuCommandBuilder { vm: &vm, diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index eab200244..19d72118a 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -11,16 +11,16 @@ use dstack_vmm_rpc as pb; use fs_err as fs; use supervisor_client::supervisor::ProcessInfo; -use super::{ - network::{mac_address_for_vm_index, resolved_networks}, - Manifest, VmState, VmWorkDir, -}; -use crate::config::{CvmConfig, GatewayConfig, Networking, NetworkingMode}; +use super::{network::mac_address_for_vm_index, Manifest, VmState, VmWorkDir}; +use crate::config::{GatewayConfig, Networking, NetworkingMode, NicNetworking}; pub(crate) struct VmInfo { pub manifest: Manifest, pub workdir: PathBuf, pub status: &'static str, + /// Whether a QEMU process exists for this VM right now. The NICs it built + /// are real only while it does. + pub running: bool, pub uptime: String, pub exited_at: Option, pub instance_id: Option, @@ -51,16 +51,83 @@ fn networking_backend_name(mode: NetworkingMode) -> &'static str { } } -fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { +/// The resolved NICs a launch built, or would build, as the RPC reports them. +fn interfaces_to_proto( + vm_id: &str, + effective_networks: &[Networking], +) -> Vec { + effective_networks + .iter() + .enumerate() + .map(|(index, networking)| { + let mac = mac_address_for_vm_index(vm_id, &networking.mac_prefix_bytes(), index); + pb::NetworkInterfaceStatus { + mode: networking_mode_name(networking.nic.mode).into(), + backend: networking_backend_name(networking.nic.mode).into(), + mac, + bridge_name: (networking.nic.mode == NetworkingMode::Bridge) + .then(|| networking.nic.bridge.clone()), + netdev_id: Some(format!("net{index}")), + // Custom mode hands the operator the whole netdev string and the + // VMM never parses it, so it has no data-plane state to report. + // Reporting the resolved fields anyway would assert "vhost: off, + // queues: 1" over a netdev the operator may have written with + // `vhost=on,queues=8`. + // + // Otherwise: settled at launch, so an entry carrying no decision + // was written before this VMM recorded one -- by a build that + // had no vhost at all, which is what it should read as. + // Recomputing here instead would let an edit to node + // configuration change what a running VM is said to use. + vhost: (networking.nic.mode != NetworkingMode::Custom) + .then(|| networking.nic.vhost.is_some() && networking.vhost_enabled()), + queues: (networking.nic.mode != NetworkingMode::Custom) + .then(|| networking.queue_pairs()), + // Node-decided, so it belongs with the rest of the resolved + // state. The VM's own record cannot carry one. + macvtap_mode: (networking.nic.mode == NetworkingMode::Macvtap) + .then(|| networking.macvtap_mode.clone()), + } + }) + .collect() +} + +pub(crate) fn networking_to_proto(networking: &NicNetworking) -> pb::NetworkingConfig { + // An entry that inherited its backend reports no mode, so it must report + // none of the fields that only make sense alongside one: a mode-less + // override carrying, say, a parent is something the deployment RPC + // rejects, which would strand the VM's tuning as uneditable. + let pins_backend = !networking.inherit_mode; pb::NetworkingConfig { - mode: networking_mode_name(networking.mode).into(), - bridge_name: if networking.mode == NetworkingMode::Bridge { + // An entry that only tuned the data plane named no backend, and the + // deployment RPC spells that as an empty mode. Reporting the node's + // current mode here would turn a read-modify-write into a request to + // pin it -- which policy may not even permit the caller to make. + mode: if networking.inherit_mode { + String::new() + } else { + networking_mode_name(networking.mode).into() + }, + bridge_name: if pins_backend && networking.mode == NetworkingMode::Bridge { networking.bridge.clone() } else { String::new() }, - parent: networking.parent.clone(), - macvtap_mode: networking.macvtap_mode.clone(), + // Scope the macvtap fields to macvtap, the way bridge_name is scoped to + // bridge. Reporting an inherited parent on a bridge NIC produced a + // configuration that could be read but not sent back: the deployment + // RPC rejects `parent` outside macvtap mode. + parent: if pins_backend && networking.mode == NetworkingMode::Macvtap { + networking.parent.clone() + } else { + String::new() + }, + // The forwarding mode is node-controlled, so a VM never pins one and + // the type it stores can no longer carry one. It stays on the wire + // because the deployment RPC still has to reject a caller that sets it. + macvtap_mode: String::new(), + vhost: networking.vhost, + queues: networking.queues, } } @@ -69,15 +136,20 @@ fn sanitize_optional>(value: Option) -> Option { } impl VmInfo { - pub fn effective_networks(&self, cvm: &CvmConfig) -> Vec { - if self.runtime_networks.is_empty() { - resolved_networks(&self.manifest, cvm) - } else { - self.runtime_networks.clone() - } - } - - pub fn to_pb(&self, gateway: &GatewayConfig, cvm: &CvmConfig, brief: bool) -> pb::VmInfo { + /// Takes no `CvmConfig` on purpose. Everything it reports about a VM's + /// data plane was decided when that VM launched and written into + /// `effective_networks`; consulting node configuration here is what let an + /// operator's edit change what a running VM was said to be using. + /// + /// `effective_networks` is passed in rather than derived for the same + /// reason, plus one more: a stopped VM's NICs are a prediction, and only + /// the caller can consult netd to make the prediction its launch would. + pub fn to_pb( + &self, + gateway: &GatewayConfig, + brief: bool, + effective_networks: &[Networking], + ) -> pb::VmInfo { let workdir = VmWorkDir::new(&self.workdir); let vm_config = workdir.manifest(); let custom_gateway_urls = vm_config @@ -91,30 +163,17 @@ impl VmInfo { .map(networking_to_proto) .collect::>(); let configured_networking = configured_networks.first().cloned(); - let interfaces = self - .effective_networks(cvm) - .iter() - .enumerate() - .map(|(index, networking)| { - let mac = mac_address_for_vm_index( - &self.manifest.id, - &networking.mac_prefix_bytes(), - index, - ); - pb::NetworkInterfaceStatus { - mode: networking_mode_name(networking.mode).into(), - backend: networking_backend_name(networking.mode).into(), - mac, - bridge_name: (networking.mode == NetworkingMode::Bridge) - .then(|| networking.bridge.clone()), - netdev_id: Some(format!("net{index}")), - } - }) - .collect(); + let interfaces = interfaces_to_proto(&self.manifest.id, effective_networks); pb::VmInfo { id: self.manifest.id.clone(), name: self.manifest.name.clone(), status: self.status.into(), + // The one predicate that says whether `interfaces` above is what a + // process built or what the next launch would build. Clients used to + // re-derive it from `status`, which answers a different question: + // a VM being removed with QEMU still up is not "running" by that + // string, yet its NICs are real. + running: self.running, uptime: self.uptime.clone(), boot_progress: self.boot_progress.clone(), boot_error: self.boot_error.clone(), @@ -261,6 +320,7 @@ impl VmState { workdir: workdir.path().to_path_buf(), instance_id, status, + running: is_running, uptime, exited_at: Some(exited_at), boot_progress: self.state.boot_progress.clone(), @@ -276,7 +336,68 @@ impl VmState { #[cfg(test)] mod tests { - use super::sanitize_optional; + use super::{interfaces_to_proto, networking_to_proto, sanitize_optional}; + use crate::config::{NetworkingMode, NicNetworking}; + + /// Custom mode hands the operator the whole netdev string and the VMM never + /// parses it, so it has no data-plane state to report. Reporting the + /// resolved defaults instead asserted "vhost off, one queue" over a netdev + /// the operator may well have written as `vhost=on,queues=8`. + #[test] + fn a_custom_netdev_reports_no_data_plane_rather_than_the_wrong_one() { + use crate::config::Networking; + + let custom = Networking { + nic: NicNetworking { + mode: NetworkingMode::Custom, + ..NicNetworking::default() + }, + netdev: "tap,id=net0,ifname=custom0,vhost=on,queues=8".into(), + ..Networking::default() + }; + let interfaces = interfaces_to_proto("vm-1", &[custom]); + assert_eq!(interfaces[0].backend, "custom"); + assert_eq!(interfaces[0].vhost, None); + assert_eq!(interfaces[0].queues, None); + + // Every other backend still answers the question. + let bridge = Networking { + nic: NicNetworking { + mode: NetworkingMode::Bridge, + vhost: Some(true), + queues: Some(4), + ..NicNetworking::default() + }, + ..Networking::default() + }; + let interfaces = interfaces_to_proto("vm-1", &[bridge]); + assert_eq!(interfaces[0].vhost, Some(true)); + assert_eq!(interfaces[0].queues, Some(4)); + } + + #[test] + fn a_reported_interface_can_be_sent_back_unchanged() { + // GetInfo output feeds UpdateVm, so anything it reports has to satisfy + // the deployment RPC's own validation. What a VM stores can no longer + // carry a node-owned field at all -- `parent` here belongs to bridge + // mode's own entry only because the type still allows both backends' + // identity fields, and reporting still scopes it to the owning mode. + let networking = NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + parent: "eth0".into(), + vhost: Some(false), + queues: Some(2), + ..NicNetworking::default() + }; + let proto = networking_to_proto(&networking); + assert_eq!(proto.mode, "bridge"); + assert_eq!(proto.bridge_name, "br0"); + assert!(proto.parent.is_empty()); + assert!(proto.macvtap_mode.is_empty()); + assert_eq!(proto.vhost, Some(false)); + assert_eq!(proto.queues, Some(2)); + } #[test] fn sanitize_optional_filters_empty_owned_values() { diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index 0b358229f..a9822c7ec 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -303,7 +303,7 @@ mod tests { let persisted = workdir.runtime_networks(); assert_eq!(persisted.len(), 1); - assert_eq!(persisted[0].parent, "br0"); + assert_eq!(persisted[0].nic.parent, "br0"); assert!(persisted[0].device.is_empty()); assert!(!fs::read_to_string(workdir.runtime_networks_path())?.contains("/dev/tap42")); fs::remove_dir_all(temp)?; diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 2e82e67c6..d7479abc4 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -357,6 +357,10 @@ pub struct CvmConfig { pub qemu_pci_hole64_size: u64, /// QEMU hotplug_off pub qemu_hotplug_off: bool, + /// Path to `qemu-bridge-helper`, used to attach an unprivileged TAP to a + /// host bridge. Empty probes the known distribution locations. + #[serde(default)] + pub qemu_bridge_helper: String, /// TDX attestation/hash scheme policy. `legacy` keeps the existing /// digest.txt measurement path; `lite` opts into split measurement CBOR; @@ -384,6 +388,12 @@ pub struct CvmConfig { #[serde(default)] pub allowed_macvtap_parents: Vec, + /// Largest virtio-net queue pair count a deployment RPC caller may request. + /// There is no node-wide count for it to bind; lowering it below the + /// scaling cap does lower the vCPU-scaled default too. + #[serde(default = "default_max_net_queues")] + pub max_net_queues: u32, + /// Optional host-side filtering for bridge interfaces. This filter does /// not apply to macvtap interfaces. #[serde(default)] @@ -592,6 +602,13 @@ pub struct NetworkFilterConfig { pub parameters: BTreeMap, } +impl NetworkFilterConfig { + /// Whether every bridge TAP on this node must carry an nwfilter binding. + pub fn requires_binding(&self) -> bool { + self.mode == NetworkFilterMode::Libvirt + } +} + impl Default for NetworkFilterConfig { fn default() -> Self { Self { @@ -615,6 +632,23 @@ pub struct NetdConfig { pub socket_mode: u32, #[serde(default = "default_libvirt_uri")] pub libvirt_uri: String, + /// The bridge filtering policy netd enforces and applies: whether a binding + /// is required, which nwfilter it names, and with what parameters. + /// + /// netd holds this itself rather than taking it from each request. It is + /// the privileged side of the socket, and a caller that chose the filter + /// could name one that drops nothing -- `allow-arp` has no drop rule at all + /// -- or pin `clean-traffic` to the gateway's MAC and IP through its + /// parameters, and still satisfy a policy that only asked for "some + /// filter". + /// + /// Unset derives it from `cvm.network_filter` in the same file, which is + /// the whole answer whenever netd and the VMM share one `vmm.toml` -- the + /// normal deployment. Set it explicitly when netd runs with a config that + /// carries no `[cvm]` section, so the daemon holding the privilege is never + /// left inferring policy from a file that does not state it. + #[serde(default)] + pub network_filter: Option, } impl Default for NetdConfig { @@ -623,11 +657,21 @@ impl Default for NetdConfig { socket: default_netd_socket(), socket_mode: 0o660, libvirt_uri: default_libvirt_uri(), + network_filter: None, } } } impl NetdConfig { + /// Resolved policy. Unset means the config named no `[cvm]` section to + /// derive it from, and an unfiltered node is the historical shape. + pub fn filter_policy(&self) -> &NetworkFilterConfig { + static UNFILTERED: std::sync::OnceLock = std::sync::OnceLock::new(); + self.network_filter + .as_ref() + .unwrap_or_else(|| UNFILTERED.get_or_init(NetworkFilterConfig::default)) + } + pub fn validate(&self) -> Result<()> { anyhow::ensure!( self.socket_mode & !0o777 == 0, @@ -715,6 +759,29 @@ impl Config { } validate_networking(&self.cvm.networking)?; + // netd creates an unfiltered TAP when the filter name is empty, which + // is what unfiltered multiqueue bridges need. Libvirt mode must never + // reach that path: it would silently produce an unbound TAP where the + // operator asked for a filtered one. + anyhow::ensure!( + self.cvm.network_filter.mode != NetworkFilterMode::Libvirt + || !self.cvm.network_filter.filter.trim().is_empty(), + "cvm.network_filter.filter must not be empty when mode is libvirt" + ); + anyhow::ensure!( + (1..=MAX_NET_QUEUES).contains(&self.cvm.max_net_queues), + "cvm.max_net_queues must be between 1 and {MAX_NET_QUEUES}" + ); + // The helper path is interpolated into QEMU's `-netdev` option list, + // which QEMU splits on ',' and '='. A path carrying either would not be + // passed through, it would end the option and start a bogus one, and the + // launch failure names neither this setting nor the file. Volume sources + // are rejected for the same reason. + anyhow::ensure!( + !self.cvm.qemu_bridge_helper.contains([',', '=']), + "cvm.qemu_bridge_helper must not contain ',' or '=': {}", + self.cvm.qemu_bridge_helper + ); anyhow::ensure!( !self .cvm @@ -828,9 +895,28 @@ fn validate_networking(networking: &Networking) -> Result<()> { "cvm.networking.mac_prefix must contain 1 to 3 two-digit hexadecimal bytes" ); } - match networking.mode { + anyhow::ensure!( + networking.nic.queues.is_none(), + "cvm.networking.queues is not a node setting; queue pairs follow each VM's vCPU count, \ + bounded by cvm.max_net_queues, and a deployment overrides them per NIC" + ); + // Both describe a per-VM entry's relationship to this configuration, so + // neither means anything on the node default itself. + anyhow::ensure!( + !networking.nic.inherit_mode, + "cvm.networking.inherit_mode is per-deployment state and cannot be set on the node default" + ); + anyhow::ensure!( + networking.netd_interface.is_none(), + "cvm.networking.netd_interface is runtime state and cannot be set in configuration" + ); + anyhow::ensure!( + networking.device.is_empty(), + "cvm.networking.device is runtime state and cannot be set in configuration" + ); + match networking.nic.mode { NetworkingMode::Bridge => anyhow::ensure!( - !networking.bridge.trim().is_empty(), + !networking.nic.bridge.trim().is_empty(), "cvm.networking.bridge must not be empty in bridge mode" ), NetworkingMode::Custom => anyhow::ensure!( @@ -839,7 +925,7 @@ fn validate_networking(networking: &Networking) -> Result<()> { ), NetworkingMode::Macvtap => { anyhow::ensure!( - !networking.parent.trim().is_empty(), + !networking.nic.parent.trim().is_empty(), "cvm.networking.parent must not be empty in macvtap mode" ); anyhow::ensure!( @@ -850,6 +936,7 @@ fn validate_networking(networking: &Networking) -> Result<()> { "cvm.networking.macvtap_mode must be private, bridge, vepa, or passthru" ); } + // User mode has no identity fields of its own to check. NetworkingMode::User => {} } Ok(()) @@ -859,20 +946,49 @@ fn default_allowed_network_modes() -> Vec { vec![NetworkingMode::User, NetworkingMode::Bridge] } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +fn default_max_net_queues() -> u32 { + DEFAULT_MAX_NET_QUEUES +} + +/// Where the vCPU-scaled default stops growing. Each queue pair costs a host +/// vhost thread and two MSI-X vectors, and cross-vCPU wakeups are expensive +/// under TDX, so the benefit runs out well before a large VM's vCPU count. +/// Raising `cvm.max_net_queues` lets a deployment ask for more; it does not +/// move this, because a bigger VM should not silently get a worse default. +pub const DEFAULT_QUEUE_SCALING_CAP: u32 = 16; + +/// Default ceiling on what a deployment RPC caller may request. +pub const DEFAULT_MAX_NET_QUEUES: u32 = 16; + +/// Hard bound on queue pairs from any source, well below anything QEMU or the +/// guest driver would refuse. It exists so a malformed or hostile request +/// cannot ask the host kernel for an unbounded device, not because 64 is a +/// property of virtio-net. +pub const MAX_NET_QUEUES: u32 = 64; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "lowercase")] pub enum NetworkingMode { + /// The backend that needs nothing from the host, so it is what a NIC that + /// names none falls back to. + #[default] User, Bridge, Custom, Macvtap, } -/// Flat networking configuration. The `mode` field selects which backend is -/// active; the remaining fields are only relevant for their respective mode -/// and carry serde defaults so they can be omitted in the config file. -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct Networking { +/// What a single NIC pins: the fields a deployment may name, a VM's manifest +/// stores, and `GetInfo` reports back. +/// +/// Separate from [`Networking`] because the node's `[cvm.networking]` is not a +/// NIC -- it is a NIC *plus* the backend settings only the node may set. When +/// the two were one type, resolution copied the whole node value into every +/// VM, so a bridge NIC's manifest entry carried whatever macvtap parent the +/// node happened to have configured, and every consumer that asked "what does +/// this VM pin?" had to know which fields to ignore. Several did not. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct NicNetworking { pub mode: NetworkingMode, // ── Bridge fields ────────────────────────────────────────────── @@ -884,6 +1000,44 @@ pub struct Networking { /// Parent host interface for macvtap (e.g., "eth0"). #[serde(default)] pub parent: String, + + // ── Data plane tuning ────────────────────────────────────────── + /// Move packet processing from the QEMU main loop into the host kernel's + /// vhost-net data plane. `None` inherits the node default. Ignored by the + /// user-mode backend, which has no vhost support, and by custom mode, + /// which owns its whole netdev string. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vhost: Option, + /// virtio-net queue pairs. `None` scales with the VM's vCPU count. Only a + /// deployment sets this; there is no node-wide value, because the useful + /// number depends on the VM rather than the host. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queues: Option, + + // ── Ownership markers ────────────────────────────────────────── + /// Take `mode` from node configuration at every launch instead of from + /// this entry. + /// + /// A deployment that only tunes the data plane never named a backend, so + /// the node still owns which one this NIC uses. `mode` is not an `Option` + /// -- every consumer matches on it -- so the entry carries the node's + /// current mode and this flag says not to trust it across a node + /// configuration change. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub inherit_mode: bool, +} + +/// `[cvm.networking]`, and the resolved value a launch hands to QEMU: one NIC +/// plus the backend settings that belong to the node rather than to any VM. +/// +/// Resolution produces this same type because a launch needs both halves; what +/// it must never do is hand the node half back to a VM to store. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct Networking { + #[serde(flatten)] + pub nic: NicNetworking, + + // ── Macvtap fields ──────────────────────────────────────────── /// macvtap forwarding mode. Empty selects "private". #[serde(default)] pub macvtap_mode: String, @@ -908,11 +1062,108 @@ pub struct Networking { // ── Custom fields ────────────────────────────────────────────── #[serde(default)] pub netdev: String, + + // ── Runtime markers ──────────────────────────────────────────── + /// What netd built for this NIC, recorded when it was built. + /// + /// Runtime state, like `device`: resolution always clears it. Teardown + /// reads this rather than re-deriving it from node configuration, because + /// an operator may change `network_filter.mode` or `max_net_queues` while + /// the VM runs, and what has to be removed is what was created. + #[serde(default, skip_serializing_if = "NetdInterface::is_none")] + pub netd_interface: NetdInterface, +} + +/// The host interface netd created for a NIC, if any. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum NetdInterface { + /// netd was not involved: user mode, custom mode, or a bridge NIC that + /// QEMU's own bridge helper attaches. + #[default] + None, + /// netd created the interface and bound no libvirt nwfilter to it. + Unfiltered, + /// netd created the interface and bound a libvirt nwfilter to it, which + /// removal has to delete before the interface goes away. + Filtered, +} + +impl NetdInterface { + pub fn is_none(&self) -> bool { + matches!(self, NetdInterface::None) + } + + pub fn is_filtered(&self) -> bool { + matches!(self, NetdInterface::Filtered) + } } impl Networking { pub fn is_bridge(&self) -> bool { - self.mode == NetworkingMode::Bridge + self.nic.mode == NetworkingMode::Bridge + } + + /// Whether the vhost-net data plane applies to this interface. + /// + /// Defaults to disabled: an upgraded node keeps the exact device shape its + /// VMs booted with (one queue pair, userspace virtio) until the operator + /// opts in, because `vhost = true` requires `/dev/vhost-net` to be + /// accessible to the account QEMU runs under — a precondition the VMM + /// cannot verify on the operator's behalf. + pub fn vhost_enabled(&self) -> bool { + self.nic.vhost.unwrap_or(false) && self.supports_vhost() + } + + /// Whether the backend selected by `mode` can carry a vhost-net data plane + /// at all. Custom mode is excluded because the operator supplies the whole + /// netdev string, including any vhost options. + pub fn supports_vhost(&self) -> bool { + Self::mode_supports_vhost(self.nic.mode) + } + + /// The same question about a mode on its own, for a caller deciding + /// whether a request it has not built an entry for yet can be honoured. + pub fn mode_supports_vhost(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::Bridge | NetworkingMode::Macvtap) + } + + /// Whether the backend selected by `mode` can carry more than one queue + /// pair. + /// + /// Custom mode is excluded for the same reason as vhost: the operator + /// supplies the whole netdev string and the VMM cannot edit it, so a + /// multiqueue device line would have nothing to pair with. The RPC refuses + /// such a request, but a node that switches its default to custom must not + /// be able to produce one behind the RPC's back. + pub fn supports_multiqueue(&self) -> bool { + Self::mode_supports_multiqueue(self.nic.mode) + } + + /// The same question about a mode on its own, for a caller deciding + /// whether a request it has not built an entry for yet can be honoured. + pub fn mode_supports_multiqueue(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::Bridge | NetworkingMode::Macvtap) + } + + /// Effective virtio-net queue pair count of a resolved NIC, never below + /// one. Resolution makes the vCPU-scaled default concrete, so an entry that + /// still carries none is read conservatively as single-queue. + pub fn queue_pairs(&self) -> u32 { + if !self.supports_multiqueue() { + return 1; + } + self.nic.queues.unwrap_or(1).max(1) + } + + /// Queue pairs a VM with this many vCPUs gets when it asks for none. + /// + /// The guest driver uses at most one queue pair per vCPU, so the default + /// follows the vCPU count up to a fixed cap. A node that lowers + /// `max_net_queues` below that cap means it, so the default follows it + /// down; raising it above the cap only widens what a caller may request. + pub fn default_queue_pairs(vcpu: u32, max_net_queues: u32) -> u32 { + vcpu.clamp(1, DEFAULT_QUEUE_SCALING_CAP.min(max_net_queues).max(1)) } /// Parse the mac_prefix into bytes. Returns 0-3 bytes. @@ -1198,6 +1449,31 @@ mod tests { .expect("default VMM config should parse") } + /// The two ownership markers are additive on disk: manifests and runtime + /// network snapshots written before they existed still load, and an entry + /// that carries neither serializes exactly as it used to. + #[test] + fn ownership_markers_are_omitted_when_unset_and_default_when_absent() { + let mut networking: Networking = + serde_json::from_str(r#"{"mode":"bridge","bridge":"br0"}"#).unwrap(); + assert!(!networking.nic.inherit_mode); + assert_eq!(networking.netd_interface, NetdInterface::None); + + let json = serde_json::to_string(&networking).unwrap(); + assert!(!json.contains("inherit_mode"), "{json}"); + assert!(!json.contains("netd_interface"), "{json}"); + + networking.nic.inherit_mode = true; + networking.netd_interface = NetdInterface::Filtered; + let json = serde_json::to_string(&networking).unwrap(); + assert!(json.contains(r#""inherit_mode":true"#), "{json}"); + assert!(json.contains(r#""netd_interface":"filtered""#), "{json}"); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + networking + ); + } + #[test] fn config_validation_accepts_defaults() { let config = default_config(); @@ -1234,6 +1510,42 @@ mod tests { .to_string() .contains("range start")); + // An empty filter tells netd to create an unfiltered TAP, so libvirt + // mode must never carry one. + let mut config = default_config(); + config.cvm.network_filter.mode = NetworkFilterMode::Libvirt; + config.cvm.network_filter.filter = String::new(); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("network_filter.filter")); + + let mut config = default_config(); + config.cvm.max_net_queues = 0; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("max_net_queues")); + + let mut config = default_config(); + config.cvm.max_net_queues = MAX_NET_QUEUES + 1; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("max_net_queues")); + + // The node-wide value is gone; say so rather than ignoring it. + let mut config = default_config(); + config.cvm.networking.nic.queues = Some(4); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("cvm.networking.queues is not a node setting")); + let mut config = default_config(); config.cvm.networking.mac_prefix = "02:not-hex".into(); assert!(config @@ -1270,8 +1582,8 @@ mod tests { .contains("supervisor.sock")); let mut config = default_config(); - config.cvm.networking.mode = NetworkingMode::Bridge; - config.cvm.networking.bridge.clear(); + config.cvm.networking.nic.mode = NetworkingMode::Bridge; + config.cvm.networking.nic.bridge.clear(); assert!(config .validate() .unwrap_err() @@ -1328,3 +1640,89 @@ mod tests { assert_eq!(CvmPlatform::resolve_from_cpuinfo(cpuinfo), CvmPlatform::Tdx); } } + +#[cfg(test)] +mod networking_shape_tests { + use super::{Config, Networking, NetworkingMode, NicNetworking, DEFAULT_CONFIG}; + + /// Splitting the type must not split the wire format. `[cvm.networking]` + /// is flattened, so a node config, a stored manifest and a runtime-networks + /// snapshot written by an earlier build all still parse, and what this + /// build writes is byte-for-byte what the old one did. + #[test] + fn the_split_types_keep_one_flat_serialized_shape() { + let legacy = serde_json::json!({ + "mode": "bridge", + "bridge": "br0", + "parent": "eth0", + "macvtap_mode": "private", + "device": "/dev/tap7", + "mac_prefix": "02:aa:bb", + "net": "10.0.2.0/24", + "dhcp_start": "10.0.2.15", + "restrict": true, + "netdev": "", + "vhost": false, + "queues": 4, + "inherit_mode": true, + "netd_interface": "filtered", + }); + + // A resolved value keeps every field, at the same names as before. + let resolved: Networking = serde_json::from_value(legacy.clone()).unwrap(); + assert_eq!(resolved.nic.mode, NetworkingMode::Bridge); + assert_eq!(resolved.nic.bridge, "br0"); + assert_eq!(resolved.nic.queues, Some(4)); + assert!(resolved.nic.inherit_mode); + assert_eq!(resolved.macvtap_mode, "private"); + assert_eq!(resolved.net, "10.0.2.0/24"); + assert!(resolved.restrict); + let round_tripped = serde_json::to_value(&resolved).unwrap(); + assert_eq!(round_tripped["mode"], "bridge"); + assert_eq!(round_tripped["macvtap_mode"], "private"); + assert_eq!(round_tripped["queues"], 4); + + // A manifest entry written by a build that stored the whole thing + // still loads; the node's half is simply dropped on the way in. + let pinned: NicNetworking = serde_json::from_value(legacy).unwrap(); + assert_eq!(pinned.bridge, "br0"); + assert_eq!(pinned.parent, "eth0"); + assert_eq!(pinned.vhost, Some(false)); + let stored = serde_json::to_value(&pinned).unwrap(); + assert_eq!(stored.as_object().unwrap().len(), 6); + assert!(stored.get("macvtap_mode").is_none()); + assert!(stored.get("net").is_none()); + } + + /// The path is interpolated into QEMU's `-netdev` option list, which QEMU + /// splits on ',' and '='. A path carrying either would end the option and + /// start a bogus one, and the launch failure names neither the setting nor + /// the file. + #[test] + fn a_bridge_helper_path_cannot_end_the_qemu_option_it_sits_in() { + use rocket::figment::providers::Format as _; + let mut config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + config.cvm.qemu_bridge_helper = "/opt/qemu,helper".into(); + let error = config.validate().unwrap_err(); + assert!(error.to_string().contains("qemu_bridge_helper"), "{error}"); + + config.cvm.qemu_bridge_helper = "/usr/libexec/qemu-bridge-helper".into(); + config.validate().unwrap(); + } + + /// The TOML section still deserializes through the flatten. + #[test] + fn the_node_section_still_parses_from_toml() { + use rocket::figment::providers::Format as _; + let config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + assert_eq!(config.cvm.networking.nic.mode, NetworkingMode::User); + } +} diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 55099e81b..17b93d194 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -219,6 +219,22 @@ async fn main() -> Result<()> { if let Some(socket) = netd_args.socket.as_deref() { netd_config.socket = socket.into(); } + if netd_config.network_filter.is_none() { + // netd and the VMM normally share one vmm.toml, so the node has + // already stated whether its bridge traffic is filtered and with + // what. Reading it here keeps the two from drifting apart, which is + // what a second setting to keep in sync would invite. + // + // A malformed section is an error rather than a default: this is + // the daemon's security policy, and `[cvm.network_filter] mode = + // "Libvirt"` -- which the VMM itself refuses to start on -- must not + // quietly resolve to "filter nothing" here. + netd_config.network_filter = Some( + figment + .extract_inner("cvm.network_filter") + .context("failed to load [cvm.network_filter] for netd")?, + ); + } return netd::serve(netd_config).await; } diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 931a373ae..6f9a0905d 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -28,7 +28,7 @@ use crate::app::{ needs_swtpm, resolve_networking, validate_resolved_network, validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, }; -use crate::config::{CvmConfig, Networking, NetworkingMode}; +use crate::config::{CvmConfig, Networking, NetworkingMode, NicNetworking}; fn hex_sha256(data: &str) -> String { use sha2::Digest; @@ -352,18 +352,37 @@ fn resolve_volume_source(base: &Path, source: &str) -> Result { fn networking_from_proto( proto: &rpc::NetworkingConfig, cvm_config: &CvmConfig, -) -> Result> { +) -> Result> { let bridge = proto.bridge_name.trim().to_string(); - let mode = match proto.mode.as_str() { - "bridge" => NetworkingMode::Bridge, - "user" => NetworkingMode::User, - "macvtap" => NetworkingMode::Macvtap, - "" if bridge.is_empty() => return Ok(None), - "" => bail!("networking mode is required when bridge is set"), + let parent = proto.parent.trim().to_string(); + // Checked before anything reads `queues`, because the "no override at all" + // arm below returns early: a request of `{queues: 0}` and nothing else + // would otherwise be answered with the vCPU-scaled default -- up to + // sixteen queue pairs -- for a caller who asked for none. + if proto.queues == Some(0) { + bail!("networking queues must be at least 1, or unset to follow the vCPU count"); + } + let tuned = proto.vhost.is_some() || proto.queues.is_some(); + // Naming a bridge or a macvtap parent is naming a backend, and a backend + // needs a mode to go with it. Without one the entry would freeze whichever + // mode the node happened to have, then report a mode-less override still + // carrying a field only one mode accepts -- something nothing can send + // back once the node moves on. + let names_backend = !bridge.is_empty() || !parent.is_empty(); + // A request that only tunes the data plane keeps the node's backend. Node + // policy governs which backend a caller may *choose*, so inheriting one + // must not be denied by it. + let (mode, chosen) = match proto.mode.as_str() { + "bridge" => (NetworkingMode::Bridge, true), + "user" => (NetworkingMode::User, true), + "macvtap" => (NetworkingMode::Macvtap, true), + "" if !names_backend && !tuned => return Ok(None), + "" if !names_backend => (cvm_config.networking.nic.mode, false), + "" => bail!("networking mode is required when a bridge or macvtap parent is set"), "custom" => bail!("custom networking mode is manifest-only"), other => bail!("unsupported networking mode '{other}'"), }; - if !cvm_config.allowed_network_modes.contains(&mode) { + if chosen && !cvm_config.allowed_network_modes.contains(&mode) { bail!( "networking mode '{}' is not allowed by node policy", proto.mode @@ -372,45 +391,161 @@ fn networking_from_proto( if mode != NetworkingMode::Bridge && !bridge.is_empty() { bail!("bridge_name is only valid for bridge networking mode"); } - if mode != NetworkingMode::Macvtap && !proto.parent.trim().is_empty() { + if mode != NetworkingMode::Macvtap && !parent.is_empty() { bail!("parent is only valid for macvtap networking mode"); } - if !proto.macvtap_mode.trim().is_empty() { + // `GetInfo` reports the resolved node defaults, and both vmm-cli and the + // web UI read that, change one field, and send the rest back. Naming a + // value the node would have supplied anyway therefore has to be accepted: + // leaving the field empty already yields exactly it, so echoing it grants + // nothing that policy was withholding. + // Node values are compared trimmed, the way the request's are: node + // configuration validation tolerates surrounding whitespace, and a value + // resolution would supply must not become unsendable over a space. + let node = &cvm_config.networking; + let macvtap_mode = proto.macvtap_mode.trim(); + if !macvtap_mode.is_empty() && macvtap_mode != node.macvtap_mode.trim() { bail!("macvtap_mode is node-controlled and cannot be set by deployment RPCs"); } - if !bridge.is_empty() && !cvm_config.allowed_bridges.contains(&bridge) { + if !bridge.is_empty() + && bridge != node.nic.bridge.trim() + && !cvm_config.allowed_bridges.contains(&bridge) + { bail!("bridge_name '{bridge}' is not allowed by node policy"); } - let parent = proto.parent.trim().to_string(); - if !parent.is_empty() && !cvm_config.allowed_macvtap_parents.contains(&parent) { + if !parent.is_empty() + && parent != node.nic.parent.trim() + && !cvm_config.allowed_macvtap_parents.contains(&parent) + { bail!("macvtap parent '{parent}' is not allowed by node policy"); } - Ok(Some(Networking { + // Same rule as the queue count below: a backend the caller chose and that + // has no vhost data plane is a request they can fix, so say so. An + // inherited one is not, and reads as off until the node moves. + if chosen && proto.vhost == Some(true) && !Networking::mode_supports_vhost(mode) { + bail!("{} networking has no vhost data plane", proto.mode); + } + // Queue pairs cost a host vhost thread and a pair of MSI-X vectors each, so + // the node caps what a deployment may ask for. + // + let queues = proto.queues; + if let Some(queues) = queues { + if queues > cvm_config.max_net_queues { + bail!( + "networking queues must not exceed {} on this node", + cvm_config.max_net_queues + ); + } + // Only a backend the caller *chose* is theirs to be wrong about. One + // they inherited can change under them -- that is the point of + // inheriting -- and refusing the request afterwards would strand the + // VM: GetInfo would keep reporting a queue count that nothing is + // allowed to send back. `queue_pairs()` already reads as one on both + // of these, so the request simply lies dormant until the node moves to + // a backend that can honour it. + if chosen && queues > 1 && !Networking::mode_supports_multiqueue(mode) { + bail!("{} networking does not support multiple queues", proto.mode); + } + } + // Every field the node owns -- the macvtap forwarding mode, the MAC prefix, + // the user-mode network parameters, a custom netdev string -- is absent by + // construction now rather than by remembering to write `String::new()` for + // each of them. + Ok(Some(NicNetworking { mode, + // The caller named no backend, so the node keeps owning which one this + // NIC uses. `mode` above is only the node's current choice. + inherit_mode: !chosen, bridge, parent, - // The forwarding mode is always inherited from node configuration. - macvtap_mode: String::new(), - device: String::new(), - mac_prefix: String::new(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, - netdev: String::new(), + vhost: proto.vhost, + queues, })) } +/// Networking modes a client should offer. +/// +/// A mode the host could serve but node policy forbids is not one of them: +/// offering it puts a choice in the deploy dialog whose only outcome is "not +/// allowed by node policy", with nothing for the operator to do about it. A VM +/// can still be given a data plane override without naming any mode, which is +/// how a node whose own backend is not caller-selectable stays tunable. +fn advertised_modes(cvm_config: &CvmConfig, host_can_bridge: bool) -> Vec { + [ + (NetworkingMode::User, "user", true), + (NetworkingMode::Bridge, "bridge", host_can_bridge), + (NetworkingMode::Macvtap, "macvtap", true), + ] + .into_iter() + .filter(|(mode, _, host_supports)| { + *host_supports && cvm_config.allowed_network_modes.contains(mode) + }) + .map(|(_, name, _)| name.to_string()) + .collect() +} + +/// The node's policy, widened by what this VM's own NICs already pin. +/// +/// Deployment allowlists govern what a caller may *newly* select. A value one +/// of this VM's interfaces is already running on was selected when it was +/// still allowed, and it is reported back on every `GetInfo`; refusing it +/// would strand the VM rather than withhold anything. +fn held_networking_config(cvm_config: &CvmConfig, held: &[NicNetworking]) -> CvmConfig { + let mut widened = cvm_config.clone(); + for networking in held { + // Only a field the entry's own mode owns. Resolution starts from the + // node's whole `[cvm.networking]` value, so a bridge entry also carries + // whatever macvtap parent the node happened to have configured, and a + // macvtap entry carries its bridge -- baggage the VM never used. Reading + // those as "already running on it" would widen policy from a value + // nothing ever attached to, and let a VM move to a backend the node + // forbids. + if networking.mode == NetworkingMode::Bridge && !networking.bridge.is_empty() { + widened.allowed_bridges.push(networking.bridge.clone()); + } + if networking.mode == NetworkingMode::Macvtap && !networking.parent.is_empty() { + widened + .allowed_macvtap_parents + .push(networking.parent.clone()); + } + // Same rule for the queue cap. Lowering `max_net_queues` bounds what a + // deployment may newly ask for; it does not retroactively re-tune VMs + // that are already pinned above it, and those keep reporting their + // count on every `GetInfo`. Without this, lowering the cap makes every + // networking update on such a VM fail on a field the operator never + // typed -- including the `--net-queues auto` that would unpin it. + if let Some(queues) = networking.queues { + widened.max_net_queues = widened.max_net_queues.max(queues); + } + } + widened +} + +/// A NIC that overrides nothing: whatever the node's `[cvm.networking]` says, +/// now and after the operator changes it. +fn node_default_networking(cvm_config: &CvmConfig) -> NicNetworking { + NicNetworking { + mode: cvm_config.networking.nic.mode, + inherit_mode: true, + ..NicNetworking::default() + } +} + fn network_from_required_proto( proto: &rpc::NetworkingConfig, cvm_config: &CvmConfig, -) -> Result { - networking_from_proto(proto, cvm_config)?.context("networking mode is required") +) -> Result { + // An entry in a list that overrides nothing is not a missing mode: it is a + // NIC that follows the node entirely. Only the singular `networking` field + // can mean "no override at all", because there the absence is the message. + Ok(networking_from_proto(proto, cvm_config)? + .unwrap_or_else(|| node_default_networking(cvm_config))) } fn networks_from_proto( networks: &[rpc::NetworkingConfig], cvm_config: &CvmConfig, -) -> Result> { +) -> Result> { networks .iter() .map(|network| network_from_required_proto(network, cvm_config)) @@ -422,15 +557,68 @@ fn validate_default_network(cvm_config: &CvmConfig) -> Result<()> { } fn resolve_requested_networks( - networks: &[Networking], + requests: &[NicNetworking], cvm_config: &CvmConfig, -) -> Result> { - let resolved = networks + vcpu: u32, +) -> Result> { + let merged = requests .iter() - .map(|networking| resolve_networking(networking, cvm_config)) + .map(|request| resolve_networking(request, cvm_config, vcpu)) .collect::>(); - validate_resolved_networks(&resolved)?; - Ok(resolved) + // Validate the merged view, because that is what the launch sees, then + // record the narrower view the manifest keeps. + validate_resolved_networks(&merged)?; + Ok(manifest_networks(merged, requests)) +} + +/// What a deployment records against the VM, given the merged view its launch +/// would see. +/// +/// The backend a deployment *chose* is pinned here -- its mode, and the bridge +/// or macvtap parent that names it -- so a VM keeps the segment it was put on +/// for life. Everything else stays owned by the node and is re-read at every +/// launch: the MAC prefix, the user-mode subnet and DHCP start, the macvtap +/// forwarding mode, and the data plane an operator may want to roll back +/// node-wide. A VM's own record cannot carry any of those. +fn manifest_networks(merged: Vec, requests: &[NicNetworking]) -> Vec { + merged + .into_iter() + .zip(requests) + .map(|(entry, request)| { + if request.inherit_mode { + // The caller named no backend, so none of the node's matching + // identity fields are theirs to keep. + return request.clone(); + } + // Taking `entry.nic` rather than the whole resolved value is what + // keeps the node's half out of the VM's record; it used to take the + // whole thing and then clear the one node field anybody noticed. + // + // The two identity fields still share one type, and resolution + // fills both from the node, so scope them here -- once, where the + // record is written -- rather than leaving every later reader to + // remember which one its mode owns. Two already have to. + let mode = entry.nic.mode; + NicNetworking { + bridge: if mode == NetworkingMode::Bridge { + entry.nic.bridge + } else { + String::new() + }, + parent: if mode == NetworkingMode::Macvtap { + entry.nic.parent + } else { + String::new() + }, + // Data plane tuning is not identity: leave what the caller did + // not ask for unset. + vhost: request.vhost, + queues: request.queues, + mode, + inherit_mode: entry.nic.inherit_mode, + } + }) + .collect() } fn has_host_bridge_interface() -> bool { @@ -445,13 +633,13 @@ fn has_host_bridge_interface() -> bool { fn networks_from_vm_config( request: &VmConfiguration, cvm_config: &CvmConfig, -) -> Result> { +) -> Result> { if !request.networks.is_empty() { let networks = networks_from_proto(&request.networks, cvm_config)?; - resolve_requested_networks(&networks, cvm_config) + resolve_requested_networks(&networks, cvm_config, request.vcpu) } else if let Some(networking) = request.networking.as_ref() { match networking_from_proto(networking, cvm_config)? { - Some(networking) => resolve_requested_networks(&[networking], cvm_config), + Some(networking) => resolve_requested_networks(&[networking], cvm_config, request.vcpu), None => Ok(vec![]), } } else { @@ -747,8 +935,14 @@ impl VmmRpc for RpcHandler { validate_default_network(&self.app.config.cvm)?; vec![] } else { - let networks = networks_from_proto(&request.networks, &self.app.config.cvm)?; - resolve_requested_networks(&networks, &self.app.config.cvm)? + // A bridge or parent this VM already holds is not a new grant. + // `GetInfo` keeps reporting it, and read-modify-write keeps + // sending it back, so refusing it once the node changes its own + // default would make the VM's configuration unsendable -- with + // no flag anywhere to clear a field the caller never typed. + let cvm = held_networking_config(&self.app.config.cvm, &manifest.networks); + let networks = networks_from_proto(&request.networks, &cvm)?; + resolve_requested_networks(&networks, &cvm, manifest.vcpu)? }; let is_running = self .app @@ -859,14 +1053,12 @@ impl VmmRpc for RpcHandler { } async fn get_meta(self) -> Result { - let mut supported_modes = vec!["user".to_string()]; let default_networking = &self.app.config.cvm.networking; let mut bridge_networking = default_networking.clone(); - bridge_networking.mode = NetworkingMode::Bridge; - if validate_resolved_network(&bridge_networking).is_ok() || has_host_bridge_interface() { - supported_modes.push("bridge".to_string()); - } - supported_modes.push("macvtap".to_string()); + bridge_networking.nic.mode = NetworkingMode::Bridge; + let host_can_bridge = + validate_resolved_network(&bridge_networking).is_ok() || has_host_bridge_interface(); + let supported_modes = advertised_modes(&self.app.config.cvm, host_can_bridge); Ok(GetMetaResponse { kms: Some(KmsSettings { url: self @@ -900,13 +1092,15 @@ impl VmmRpc for RpcHandler { }), networking: Some(rpc::NetworkingCapabilities { supported_modes, - default_mode: match default_networking.mode { + default_mode: match default_networking.nic.mode { NetworkingMode::User => "user".to_string(), NetworkingMode::Bridge => "bridge".to_string(), NetworkingMode::Custom => String::new(), NetworkingMode::Macvtap => "macvtap".to_string(), }, - default_bridge: default_networking.bridge.clone(), + default_bridge: default_networking.nic.bridge.clone(), + max_queues: self.app.config.cvm.max_net_queues, + default_vhost: default_networking.vhost_enabled(), }), }) } @@ -1282,7 +1476,9 @@ mod tests { assert_eq!(manifest.networks.len(), 1); assert_eq!(manifest.networks[0].mode, NetworkingMode::User); - assert!(!manifest.networks[0].net.is_empty()); + // The node's user-mode network parameters are not the VM's to store; + // the type it stores cannot carry them at all now. + assert!(manifest.networks[0].bridge.is_empty()); } #[test] @@ -1292,7 +1488,7 @@ mod tests { .allowed_network_modes .push(NetworkingMode::Macvtap); cvm_config.allowed_macvtap_parents.push("eth0".to_string()); - cvm_config.networking.parent = "node-default".to_string(); + cvm_config.networking.nic.parent = "node-default".to_string(); cvm_config.networking.macvtap_mode = "private".to_string(); let networks = networks_from_proto( &[rpc::NetworkingConfig { @@ -1306,11 +1502,789 @@ mod tests { assert_eq!(networks[0].mode, NetworkingMode::Macvtap); assert_eq!(networks[0].parent, "eth0"); - assert!(networks[0].macvtap_mode.is_empty()); - let resolved = resolve_requested_networks(&networks, &cvm_config).unwrap(); - assert_eq!(resolved[0].parent, "eth0"); - assert_eq!(resolved[0].macvtap_mode, "private"); + // The manifest keeps the parent, which is identity. The forwarding mode + // the node owns and supplies at every launch is not a field this type + // has. + let stored = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + assert_eq!(stored[0].parent, "eth0"); + + let at_launch = resolve_networking(&stored[0], &cvm_config, 4); + assert_eq!(at_launch.nic.parent, "eth0"); + assert_eq!(at_launch.macvtap_mode, "private"); + + // Repointing the node's forwarding mode reaches the VM. + cvm_config.networking.macvtap_mode = "bridge".to_string(); + assert_eq!( + resolve_networking(&stored[0], &cvm_config, 4).macvtap_mode, + "bridge" + ); + } + + /// Node shapes a deployment can be reported against, each with the node + /// default fields that mode populates. + fn node_shapes() -> Vec<(&'static str, CvmConfig)> { + let mut bridge = test_cvm_config(); + bridge.networking.nic.mode = NetworkingMode::Bridge; + bridge.networking.nic.bridge = "br-node".into(); + + let mut macvtap = test_cvm_config(); + macvtap.networking.nic.mode = NetworkingMode::Macvtap; + macvtap.networking.nic.parent = "eth-node".into(); + macvtap.networking.macvtap_mode = "private".into(); + macvtap.allowed_network_modes.push(NetworkingMode::Macvtap); + + let user = test_cvm_config(); + vec![ + ("bridge node", bridge), + ("macvtap node", macvtap), + ("user node", user), + ] + } + + /// Every override a caller can express, including the tuning-only shape + /// that names no backend. + fn request_shapes(node: &CvmConfig) -> Vec<(String, rpc::NetworkingConfig)> { + let mode = networking_mode_name_for_test(node.networking.nic.mode); + // The user-mode backend has neither multiqueue nor vhost, and naming it + // and then asking for either is a refusal rather than a round-trip + // failure. Asking to turn vhost off is always legal. + let multiqueue = Networking::mode_supports_multiqueue(node.networking.nic.mode); + let vhost_on = Networking::mode_supports_vhost(node.networking.nic.mode).then_some(true); + // Naming a backend's identity field is only legal alongside a mode, so + // it varies with the named case. The node's own values are used because + // that is what GetInfo reports back. + let identity: &[(&str, &str, &str)] = &[ + ("", "", ""), + ("+ bridge", node.networking.nic.bridge.as_str(), ""), + ("+ parent", "", node.networking.nic.parent.as_str()), + ]; + let mut shapes = vec![]; + for (named, mode) in [("named", mode.to_string()), ("inherited", String::new())] { + for (tuning, vhost, queues) in [ + ("untuned", None, None), + ("vhost off", Some(false), None), + ("queues", None, multiqueue.then_some(2)), + ("both", vhost_on, multiqueue.then_some(2)), + ] { + for (label, bridge, parent) in identity { + // An inherited entry may not name a backend at all, which + // is a refusal covered by its own test. + let inherited = mode.is_empty(); + if inherited && !(bridge.is_empty() && parent.is_empty()) { + continue; + } + // Only the mode that owns a field may carry it. + let bridge_ok = node.networking.nic.mode == NetworkingMode::Bridge; + let parent_ok = node.networking.nic.mode == NetworkingMode::Macvtap; + if (!bridge.is_empty() && !bridge_ok) || (!parent.is_empty() && !parent_ok) { + continue; + } + shapes.push(( + format!("{named} + {tuning} {label}"), + rpc::NetworkingConfig { + mode: mode.clone(), + bridge_name: bridge.to_string(), + parent: parent.to_string(), + vhost, + queues, + ..Default::default() + }, + )); + } + } + } + shapes + } + + /// A backend's identity field without a mode would freeze whichever mode + /// the node had at deploy time, and then be reported alongside an empty + /// mode -- a combination the RPC itself rejects, which would leave the VM's + /// tuning permanently uneditable. + #[test] + fn an_inherited_entry_may_not_name_a_backend() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Macvtap; + cvm.networking.nic.parent = "eth-node".into(); + cvm.allowed_macvtap_parents.push("eth1".into()); + cvm.allowed_network_modes.push(NetworkingMode::Macvtap); + + let err = networking_from_proto( + &rpc::NetworkingConfig { + parent: "eth1".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap_err(); + assert!(err.to_string().contains("networking mode is required")); + + // Naming the mode alongside it is fine. + networking_from_proto( + &rpc::NetworkingConfig { + mode: "macvtap".into(), + parent: "eth1".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("a named backend is an override"); + } + + fn networking_mode_name_for_test(mode: NetworkingMode) -> &'static str { + match mode { + NetworkingMode::Bridge => "bridge", + NetworkingMode::User => "user", + NetworkingMode::Macvtap => "macvtap", + NetworkingMode::Custom => "custom", + } + } + + /// `GetInfo` reports the configuration that `UpdateVm` and `UpgradeApp` + /// take back, and both vmm-cli and the web UI read it, change one field, + /// and resend the rest. So everything reportable has to be acceptable, and + /// accepting it has to land on the same VM. + /// + /// This asserts the property over every mode and tuning combination on + /// purpose. Asserting one shape is how an inherited `parent` on a bridge + /// NIC, and then `macvtap_mode` and `bridge_name`, each reached a release: + /// every one of them was a case the fixed example did not cover. + #[test] + fn everything_get_info_reports_is_accepted_back_unchanged() { + for (node_label, cvm) in node_shapes() { + for (shape_label, request) in request_shapes(&cvm) { + let case = format!("{node_label} / {shape_label}"); + let Some(requested) = networking_from_proto(&request, &cvm) + .unwrap_or_else(|error| panic!("{case}: deployment rejected: {error:#}")) + else { + // No override at all; nothing is recorded, nothing to report. + continue; + }; + // Skip the host-dependent bridge existence check: this is + // about what the RPC reports versus what it accepts. + let merged = resolve_networking(&requested, &cvm, 4); + let stored = manifest_networks(vec![merged.clone()], &[requested]); + + let reported = crate::app::networking_to_proto(&stored[0]); + let accepted = networking_from_proto(&reported, &cvm) + .unwrap_or_else(|error| { + panic!("{case}: GetInfo output was rejected on the way back: {error:#}") + }) + .unwrap_or_else(|| panic!("{case}: the override was lost in the round trip")); + + let restored = + manifest_networks(vec![resolve_networking(&accepted, &cvm, 4)], &[accepted]); + assert_eq!(stored, restored, "{case}: round trip changed the manifest"); + assert_eq!( + resolve_networking(&restored[0], &cvm, 4), + merged, + "{case}: round trip changed what the launch sees" + ); + } + } + } + + /// A VM must not become uneditable because its node moved somewhere its + /// tuning does not apply. The report and the deployment RPC have to agree + /// on every backend the node can be pointed at, not just the one it had + /// when the VM was deployed. + #[test] + fn an_inherited_override_still_round_trips_after_the_node_moves() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-node".into(); + + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(4), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("tuning must produce an override"); + let stored = manifest_networks(vec![resolve_networking(&requested, &cvm, 8)], &[requested]); + + for mode in [ + NetworkingMode::User, + NetworkingMode::Custom, + NetworkingMode::Macvtap, + NetworkingMode::Bridge, + ] { + cvm.networking.nic.mode = mode; + cvm.networking.nic.parent = "eth-node".into(); + cvm.networking.netdev = "tap,id=net0,ifname=custom0".into(); + + let reported = crate::app::networking_to_proto(&stored[0]); + let accepted = networking_from_proto(&reported, &cvm) + .unwrap_or_else(|error| panic!("{mode:?}: report was rejected: {error:#}")) + .unwrap_or_else(|| panic!("{mode:?}: the override was lost")); + let restored = + manifest_networks(vec![resolve_networking(&accepted, &cvm, 8)], &[accepted]); + // An inherited entry still has to hold *some* mode -- every + // consumer matches on one -- and it is rewritten to whatever the + // node has now. Resolution ignores it, so compare what the launch + // sees rather than the field nothing reads. + assert!(restored[0].inherit_mode, "{mode:?}: pinned a backend"); + assert_eq!( + resolve_networking(&restored[0], &cvm, 8), + resolve_networking(&stored[0], &cvm, 8), + "{mode:?}: round trip changed what the launch sees" + ); + assert_eq!(restored[0].queues, Some(4), "{mode:?}: lost the request"); + } + } + + /// The counterpart: a node that changes its mind still reaches VMs that + /// never named a backend, and never reaches ones that did. + #[test] + fn a_node_backend_change_reaches_exactly_the_vms_that_inherited_it() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-node".into(); + + let tuning_only = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("tuning must produce an override"); + let named = networking_from_proto( + &rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("a named backend is an override"); + + let requests = vec![tuning_only, named]; + let merged = requests + .iter() + .map(|request| resolve_networking(request, &cvm, 4)) + .collect::>(); + let stored = manifest_networks(merged, &requests); + + // The operator repoints the node at a different backend. + cvm.networking.nic.mode = NetworkingMode::User; + assert_eq!( + resolve_networking(&stored[0], &cvm, 4).nic.mode, + NetworkingMode::User, + "a VM that never named a backend must follow the node" + ); + assert_eq!( + resolve_networking(&stored[1], &cvm, 4).nic.mode, + NetworkingMode::Bridge, + "a VM that named its backend keeps it for life" + ); + // User mode has no multiqueue backend, so the inherited NIC drops to + // one queue pair while it is there -- but the request is not lost. + assert_eq!(resolve_networking(&stored[0], &cvm, 4).queue_pairs(), 1); + assert_eq!(stored[0].queues, Some(2)); + cvm.networking.nic.mode = NetworkingMode::Bridge; + assert_eq!( + resolve_networking(&stored[0], &cvm, 4).queue_pairs(), + 2, + "tuning must survive an excursion through a backend that ignores it" + ); + } + + #[test] + fn queue_requests_are_bounded_by_node_policy() { + let mut cvm_config = test_cvm_config(); + cvm_config.max_net_queues = 4; + cvm_config.allowed_bridges.push("tenant-br0".to_string()); + let request = |queues: u32| { + [rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "tenant-br0".to_string(), + queues: Some(queues), + ..Default::default() + }] + }; + + let networks = networks_from_proto(&request(4), &cvm_config).unwrap(); + assert_eq!(networks[0].queues, Some(4)); + + let err = networks_from_proto(&request(5), &cvm_config).unwrap_err(); + assert!(err.to_string().contains("must not exceed 4")); + } + + /// A mode the deploy dialog offers has to be one the deployment RPC will + /// take. Offering one node policy forbids puts a choice in front of an + /// operator whose only outcome is "not allowed by node policy". + #[test] + fn advertised_modes_are_ones_the_rpc_would_accept() { + let mut cvm = test_cvm_config(); + cvm.allowed_network_modes = vec![NetworkingMode::User]; + for mode in ["bridge", "macvtap"] { + assert!( + networking_from_proto( + &rpc::NetworkingConfig { + mode: mode.to_string(), + ..Default::default() + }, + &cvm, + ) + .is_err(), + "{mode} should be refused by this policy" + ); + } + assert_eq!(advertised_modes(&cvm, true), vec!["user".to_string()]); + + cvm.allowed_network_modes = vec![NetworkingMode::User, NetworkingMode::Macvtap]; + assert_eq!( + advertised_modes(&cvm, true), + vec!["user".to_string(), "macvtap".to_string()] + ); + + // A mode policy allows but the host cannot serve is still not offered. + cvm.allowed_network_modes.push(NetworkingMode::Bridge); + assert!(!advertised_modes(&cvm, false).contains(&"bridge".to_string())); + assert!(advertised_modes(&cvm, true).contains(&"bridge".to_string())); + } + + /// A VM keeps its bridge for life, so the node dropping that bridge from + /// its own configuration must not make the VM's reported configuration + /// unsendable -- there is no flag anywhere to clear a field the caller + /// never typed. + /// Deployment allowlists govern what a caller may newly select. Widening + /// them from a VM's own holdings is what keeps read-modify-write working + /// across a node change -- but only from a field the entry's own mode + /// owns. Resolution used to copy the node's whole networking value into + /// every VM, so a bridge NIC carried whatever macvtap parent the node had + /// configured, and widening from that let the VM move to a parent policy + /// forbids. The split types make the node's other fields unreachable; the + /// two identity fields still share one type, so the scoping stays explicit. + #[test] + fn holdings_widen_policy_only_for_the_mode_that_owns_them() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.parent = "eth1".into(); + cvm.allowed_network_modes = vec![ + NetworkingMode::Bridge, + NetworkingMode::Macvtap, + NetworkingMode::User, + ]; + assert!(cvm.allowed_macvtap_parents.is_empty()); + + // What a bridge NIC deployed on this node used to end up holding. + let held = [NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + parent: "eth0".into(), + ..NicNetworking::default() + }]; + let widened = held_networking_config(&cvm, &held); + assert!(widened.allowed_bridges.contains(&"br0".to_string())); + assert!(!widened + .allowed_macvtap_parents + .contains(&"eth0".to_string())); + + let request = [rpc::NetworkingConfig { + mode: "macvtap".into(), + parent: "eth0".into(), + ..Default::default() + }]; + let err = networks_from_proto(&request, &widened).unwrap_err(); + assert!( + err.to_string().contains("not allowed by node policy"), + "{err}" + ); + } + + /// Lowering the node's queue cap bounds what a deployment may newly ask + /// for. It does not retune VMs already pinned above it, and those keep + /// reporting their count on every GetInfo -- so without this, lowering the + /// cap makes every networking update on such a VM fail on a field the + /// operator never typed, including the one that would unpin it. + #[test] + fn a_vm_may_restate_a_queue_count_the_node_has_since_capped() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.max_net_queues = 2; + + let held = [NicNetworking { + mode: NetworkingMode::Bridge, + queues: Some(8), + ..NicNetworking::default() + }]; + let request = [rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(8), + vhost: Some(false), + ..Default::default() + }]; + + let err = networks_from_proto(&request, &cvm).unwrap_err(); + assert!(err.to_string().contains("must not exceed 2"), "{err}"); + + let widened = held_networking_config(&cvm, &held); + let networks = networks_from_proto(&request, &widened).unwrap(); + assert_eq!(networks[0].queues, Some(8)); + + // Only up to what it holds, though. + let more = [rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(9), + ..Default::default() + }]; + let err = networks_from_proto(&more, &widened).unwrap_err(); + assert!(err.to_string().contains("must not exceed 8"), "{err}"); + } + + /// `optional uint32` tells an absent field from a typed zero, so reading + /// zero as "unset" would answer a request for no queues with the + /// vCPU-scaled default. Resize says the same about a zero vCPU count. + #[test] + fn an_explicit_zero_queue_count_is_an_error_not_an_absent_field() { + let cvm = test_cvm_config(); + let request = rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(0), + ..Default::default() + }; + let err = networking_from_proto(&request, &cvm).unwrap_err(); + assert!(err.to_string().contains("must be at least 1"), "{err}"); + + // Including when it is the only thing the request says. That arm + // returns "no override at all" before anything reads the count, so a + // zero here used to be answered with up to sixteen queue pairs. + let bare = rpc::NetworkingConfig { + queues: Some(0), + ..Default::default() + }; + let err = networking_from_proto(&bare, &cvm).unwrap_err(); + assert!(err.to_string().contains("must be at least 1"), "{err}"); + } + + /// Resolution fills both identity fields from the node, so a bridge NIC's + /// resolved value carries whatever macvtap parent the node happens to have + /// configured. Storing that made every later reader responsible for knowing + /// which field its mode owns, and the one that widens deployment policy + /// from a VM's holdings got it wrong: repoint the node's parent and a + /// bridge VM could move itself to a parent policy forbids. + #[test] + fn a_vm_records_only_the_identity_field_its_own_mode_owns() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-node".into(); + cvm.networking.nic.parent = "eth-node".into(); + + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "bridge".into(), + ..Default::default() + }], + &cvm, + ) + .unwrap(); + // `manifest_networks` directly: what a VM records is the question, and + // `resolve_requested_networks` would first validate the merged view + // against this host's real interfaces. + let merged = networks + .iter() + .map(|request| resolve_networking(request, &cvm, 4)) + .collect::>(); + let stored = manifest_networks(merged, &networks); + assert_eq!(stored[0].bridge, "br-node"); + assert!(stored[0].parent.is_empty(), "{:?}", stored[0]); + + // And the launch still gets the node's parent, because that half was + // never the VM's to hold in the first place. + let at_launch = resolve_networking(&stored[0], &cvm, 4); + assert_eq!(at_launch.nic.parent, "eth-node"); + } + + #[test] + fn a_vm_may_restate_a_bridge_it_already_holds() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-new".into(); + assert!(cvm.allowed_bridges.is_empty()); + + let held = [NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br-old".into(), + ..NicNetworking::default() + }]; + let request = [rpc::NetworkingConfig { + mode: "bridge".into(), + bridge_name: "br-old".into(), + queues: Some(2), + ..Default::default() + }]; + + // Without the VM's own holdings this is a bridge it may not select. + let err = networks_from_proto(&request, &cvm).unwrap_err(); + assert!(err.to_string().contains("not allowed by node policy")); + + let widened = held_networking_config(&cvm, &held); + let networks = networks_from_proto(&request, &widened).unwrap(); + assert_eq!(networks[0].bridge, "br-old"); + + // And it is still only this VM's own values that are permitted. + let other = [rpc::NetworkingConfig { + mode: "bridge".into(), + bridge_name: "br-someone-else".into(), + ..Default::default() + }]; + assert!(networks_from_proto(&other, &widened).is_err()); + } + + /// vhost follows the same rule as the queue count: refused for a backend + /// the caller chose and that has none, accepted and dormant for one they + /// inherited. Accepting it silently on a chosen backend would leave the + /// deploy dialog reporting `vhost: on` next to a NIC running without it. + #[test] + fn vhost_is_refused_only_for_a_backend_the_caller_chose() { + let cvm_config = test_cvm_config(); + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + vhost: Some(true), + ..Default::default() + }, + &cvm_config, + ) + .unwrap_err(); + assert!(err.to_string().contains("no vhost data plane"), "{err:#}"); + + // Turning it off is a no-op that matches reality, so it is allowed. + networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + vhost: Some(false), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + + // Inherited from a user-mode node: accepted, dormant, and live again + // when the node moves to a backend that has one. + let mut cvm_config = test_cvm_config(); + assert_eq!(cvm_config.networking.nic.mode, NetworkingMode::User); + let requested = networking_from_proto( + &rpc::NetworkingConfig { + vhost: Some(true), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert!(!resolve_networking(&requested, &cvm_config, 4).vhost_enabled()); + + cvm_config.networking.nic.mode = NetworkingMode::Bridge; + cvm_config.networking.nic.bridge = "br-node".into(); + assert!(resolve_networking(&requested, &cvm_config, 4).vhost_enabled()); + } + + /// A queue count is refused for a backend the caller chose and that cannot + /// honour it, because the caller can fix the request. It is accepted for + /// one they inherited, because they cannot: the node picked that backend + /// and may pick another tomorrow, and refusing would leave GetInfo + /// reporting a count nothing is allowed to send back. + #[test] + fn a_queue_count_is_refused_only_for_a_backend_the_caller_chose() { + let cvm_config = test_cvm_config(); + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + queues: Some(4), + ..Default::default() + }, + &cvm_config, + ) + .unwrap_err(); + assert!(err.to_string().contains("does not support multiple queues")); + + // Inherited: accepted, and dormant until the node moves to a backend + // that can honour it. + for mode in [NetworkingMode::Custom, NetworkingMode::User] { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.mode = mode; + cvm_config.networking.netdev = "tap,id=net0,ifname=custom0".into(); + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(4), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert_eq!(requested.queues, Some(4)); + assert_eq!( + resolve_networking(&requested, &cvm_config, 8).queue_pairs(), + 1, + "{mode:?} cannot carry multiqueue, whatever was asked for" + ); + + // And the request is still there when the node moves back. + cvm_config.networking.nic.mode = NetworkingMode::Bridge; + cvm_config.networking.nic.bridge = "br-node".into(); + assert_eq!( + resolve_networking(&requested, &cvm_config, 8).queue_pairs(), + 4 + ); + } + } + + #[test] + fn user_mode_rejects_multiqueue_but_a_single_queue_is_fine() { + let cvm_config = test_cvm_config(); + let request = |queues: u32| { + [rpc::NetworkingConfig { + mode: "user".to_string(), + queues: Some(queues), + ..Default::default() + }] + }; + + networks_from_proto(&request(1), &cvm_config).unwrap(); + let err = networks_from_proto(&request(2), &cvm_config).unwrap_err(); + assert!(err.to_string().contains("does not support multiple queues")); + } + + #[test] + fn tuning_alone_keeps_the_node_backend_without_tripping_mode_policy() { + let mut cvm_config = test_cvm_config(); + // A backend the node uses but does not let callers choose. + cvm_config.networking.nic.mode = NetworkingMode::Macvtap; + cvm_config.networking.nic.parent = "eth0".to_string(); + assert!(!cvm_config + .allowed_network_modes + .contains(&NetworkingMode::Macvtap)); + + let networking = networking_from_proto( + &rpc::NetworkingConfig { + vhost: Some(false), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert_eq!(networking.mode, NetworkingMode::Macvtap); + assert_eq!(networking.vhost, Some(false)); + + // Naming that backend explicitly is still a choice, and still denied. + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "macvtap".to_string(), + vhost: Some(false), + ..Default::default() + }, + &cvm_config, + ) + .unwrap_err(); + assert!(err.to_string().contains("not allowed by node policy")); + + // An untouched request still means "no override at all". + assert!( + networking_from_proto(&rpc::NetworkingConfig::default(), &cvm_config) + .unwrap() + .is_none() + ); + } + + #[test] + fn an_inherited_backend_is_never_pinned_into_the_manifest() { + // Tuning must not become a way to pin a backend the caller was never + // allowed to choose, nor to freeze one the node still owns. + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.mode = NetworkingMode::Macvtap; + cvm_config.networking.nic.parent = "eth0".to_string(); + cvm_config.networking.macvtap_mode = "private".to_string(); + + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(2), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert!(requested.inherit_mode); + + let persisted = resolve_requested_networks(&[requested], &cvm_config, 4).unwrap(); + assert_eq!(persisted[0].queues, Some(2)); + // Nothing the node owns was copied in. The forwarding mode, the MAC + // prefix and the user-mode network parameters are not fields this type + // has any more; the parent is, and a NIC that named no backend does not + // get to keep the node's. + assert!(persisted[0].parent.is_empty()); + assert!(persisted[0].bridge.is_empty()); + + // Repointing the node moves the VM with it. + cvm_config.networking.nic.parent = "eth1".to_string(); + let at_launch = resolve_networking(&persisted[0], &cvm_config, 4); + assert_eq!(at_launch.nic.parent, "eth1"); + assert_eq!(at_launch.queue_pairs(), 2); + } + + #[test] + fn deployment_pins_identity_but_not_data_plane_tuning() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.vhost = Some(true); + cvm_config.networking.nic.queues = Some(2); + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "user".to_string(), + vhost: Some(false), + ..Default::default() + }], + &cvm_config, + ) + .unwrap(); + + let resolved = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + // The backend the caller chose is pinned. + assert_eq!(resolved[0].mode, NetworkingMode::User); + assert!(!resolved[0].inherit_mode); + // The explicit request is kept. + assert_eq!(resolved[0].vhost, Some(false)); + // What the caller never asked for stays unset, so the node still owns + // it: an operator disabling vhost node-wide must reach this VM too. + assert_eq!(resolved[0].queues, None); + } + + #[test] + fn a_node_wide_vhost_rollback_reaches_a_vm_deployed_with_an_override() { + // macvtap keeps this independent of which interfaces the test host has. + let mut cvm_config = test_cvm_config(); + cvm_config + .allowed_network_modes + .push(NetworkingMode::Macvtap); + cvm_config.allowed_macvtap_parents.push("eth0".to_string()); + cvm_config.networking.nic.parent = "eth0".to_string(); + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "macvtap".to_string(), + parent: "eth0".to_string(), + ..Default::default() + }], + &cvm_config, + ) + .unwrap(); + let persisted = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + assert!(persisted[0].vhost.is_none()); + + cvm_config.networking.nic.vhost = Some(false); + let at_launch = resolve_networking(&persisted[0], &cvm_config, 4); + assert!(!at_launch.vhost_enabled()); } #[test] @@ -1399,18 +2373,35 @@ mod tests { } #[test] - fn repeated_networks_rejects_empty_entries() { - let err = networks_from_proto( + /// An entry in a list that overrides nothing describes a NIC that follows + /// the node entirely -- which is a thing an operator can mean, and the + /// only way the web UI can leave a NIC's backend unpinned. Rejecting it + /// would make an inherited NIC uneditable the moment its tuning is cleared. + fn repeated_networks_accepts_an_entry_that_overrides_nothing() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.mode = NetworkingMode::Bridge; + cvm_config.networking.nic.bridge = "br-node".into(); + + let networks = networks_from_proto( &[rpc::NetworkingConfig { mode: String::new(), bridge_name: String::new(), ..Default::default() }], - &test_cvm_config(), + &cvm_config, ) - .unwrap_err(); + .unwrap(); + assert_eq!(networks.len(), 1); + assert!(networks[0].inherit_mode); + assert_eq!(networks[0].queues, None); + assert!(networks[0].bridge.is_empty()); - assert!(err.to_string().contains("networking mode is required")); + // It follows the node, like a VM with no networks at all. + cvm_config.networking.nic.mode = NetworkingMode::User; + assert_eq!( + resolve_networking(&networks[0], &cvm_config, 4).nic.mode, + NetworkingMode::User + ); } #[test] diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index b7a946133..6a760d1eb 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -5,7 +5,6 @@ //! Small privileged broker for TAP creation and libvirt nwfilter bindings. use std::{ - collections::BTreeMap, fs::{File, OpenOptions, Permissions}, io::Write as _, os::{ @@ -29,11 +28,11 @@ use tokio::{ net::{UnixListener, UnixStream}, time::timeout, }; -use tracing::{info, warn}; +use tracing::{debug, info, warn}; use uuid::Uuid; use wait_timeout::ChildExt; -use crate::config::NetdConfig; +use crate::config::{NetdConfig, NetworkFilterConfig}; const MAX_MESSAGE_SIZE: u64 = 64 * 1024; const CONNECTION_TIMEOUT: Duration = Duration::from_secs(35); @@ -41,6 +40,9 @@ const COMMAND_TIMEOUT: Duration = Duration::from_secs(30); const IP_PATH: &str = "/usr/sbin/ip"; const VIRSH_PATH: &str = "/usr/bin/virsh"; const LOCK_PATH: &str = "/run/lock/dstack-netd.lock"; +/// Upper bound on TAP queue pairs netd will create. Mirrors the VMM's own cap +/// so a malformed request cannot ask the kernel for an unbounded device. +const MAX_QUEUES: u32 = 64; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InterfaceIdentity { @@ -56,9 +58,17 @@ pub struct PrepareBridgeRequest { pub bridge: String, pub mac: String, pub qemu_uid: u32, - pub filter: String, - #[serde(default)] - pub parameters: BTreeMap, + /// Whether to bind an nwfilter to the TAP. *Which* filter, and with what + /// parameters, is netd's own configuration to decide -- a caller that named + /// them could name one that filters nothing, or pin the binding to the + /// gateway's MAC and IP, and still satisfy a node policy that only asked + /// for "some filter". An unfiltered TAP is what multiqueue bridge + /// networking needs on nodes that do not run libvirt. + pub filtered: bool, + /// virtio-net queue pairs. Zero or one creates a single-queue TAP. QEMU + /// rejects a device whose `IFF_MULTI_QUEUE` state differs from its own + /// `queues=` argument, so this must match the launch exactly. + pub queues: u32, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -70,6 +80,10 @@ pub struct PrepareMacvtapRequest { pub qemu_uid: u32, #[serde(default)] pub mode: String, + /// virtio-net queue pairs. The device is created with matching hardware + /// queues; QEMU then opens the character device once per queue. + #[serde(default)] + pub queues: u32, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -80,12 +94,17 @@ pub enum Request { Remove { #[serde(flatten)] identity: InterfaceIdentity, + /// Whether this interface was created with an nwfilter binding. + /// Macvtap TAPs never carry one, and removal detects them rather than + /// trusting this field. + filtered: bool, }, /// Verify a deterministic TAP and binding for operations and integration /// diagnostics. The VMM startup path uses Prepare rather than Check. Check { #[serde(flatten)] identity: InterfaceIdentity, + filtered: bool, }, } @@ -96,10 +115,33 @@ struct Response { tap: Option, #[serde(default, skip_serializing_if = "Option::is_none")] device: Option, + /// Queue pairs the interface was actually created with. Absent from a netd + /// that predates multiqueue, which is how the VMM tells the difference + /// between "one queue was requested" and "this netd ignored the request". + #[serde(default, skip_serializing_if = "Option::is_none")] + queues: Option, #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, } +/// What netd built, echoed back so the caller can verify it matches the +/// request before handing the interface to QEMU. +struct Prepared { + tap: String, + device: Option, + queues: Option, +} + +impl Prepared { + fn tap(tap: String) -> Self { + Self { + tap, + device: None, + queues: None, + } + } +} + pub fn tap_name(identity: &InterfaceIdentity) -> String { let input = format!( "{}\0{}\0{}", @@ -119,6 +161,28 @@ pub fn instance_id(configured: &str, run_path: &Path) -> String { pub struct PreparedInterface { pub device: Option, + pub queues: Option, +} + +/// Marker carried in the error chain when the VMM could not reach netd at all. +/// +/// "netd refused this" and "netd is not there" call for different advice, and +/// the caller cannot tell them apart from the message alone -- a callback probe +/// afterwards would answer about a different moment. +#[derive(Debug)] +pub struct Unreachable; + +impl std::fmt::Display for Unreachable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("netd is not reachable") + } +} + +impl std::error::Error for Unreachable {} + +/// Whether this error means netd was never reached. +pub fn is_unreachable(error: &anyhow::Error) -> bool { + error.chain().any(|cause| cause.is::()) } pub async fn request(socket: &Path, request: &Request) -> Result { @@ -131,6 +195,8 @@ pub async fn request(socket: &Path, request: &Request) -> Result MAX_MESSAGE_SIZE { @@ -154,11 +220,10 @@ pub async fn request(socket: &Path, request: &Request) -> Result Result { async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Result<()> { // Access is authorized by the Unix socket's owner, group, and mode. Any // process that can connect is trusted with the complete netd protocol. - let response = match read_request(stream) - .await - .and_then(|request| handle_request(&config.libvirt_uri, request)) - { - Ok((tap, device)) => Response { + let outcome = match read_request(stream).await { + // A peer that connects and closes without sending is the VMM's + // reachability check: netd that died leaves its socket behind, so the + // VMM connects to tell the two apart. Answering that with a parse error + // and a warning would fill the log with reports of it working. + Ok(None) => { + debug!("netd liveness probe"); + return Ok(()); + } + Ok(Some(request)) => handle_request(config, request), + // A request that arrived but could not be understood still gets an + // answer. A VMM newer than this netd sends operations it does not + // know, and "unknown variant `prepare_foo`" is what tells the operator + // to upgrade; a closed connection tells them nothing. + Err(error) => Err(error), + }; + let response = match outcome { + Ok(prepared) => Response { ok: true, - tap: Some(tap), - device, + tap: Some(prepared.tap), + device: prepared.device, + queues: prepared.queues, error: None, }, Err(error) => { @@ -241,6 +320,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul ok: false, tap: None, device: None, + queues: None, error: Some(format!("{error:#}")), } } @@ -251,77 +331,105 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul Ok(()) } -async fn read_request(stream: &mut UnixStream) -> Result { +/// Reads one request, or `None` if the peer closed without sending anything. +async fn read_request(stream: &mut UnixStream) -> Result> { let mut message = Vec::new(); stream .take(MAX_MESSAGE_SIZE + 1) .read_to_end(&mut message) .await?; + if message.is_empty() { + return Ok(None); + } if message.len() as u64 > MAX_MESSAGE_SIZE { bail!("request exceeds {MAX_MESSAGE_SIZE} bytes"); } - serde_json::from_slice(&message).context("invalid netd request") + serde_json::from_slice(&message) + .map(Some) + .context("invalid netd request") } -fn handle_request(libvirt_uri: &str, request: Request) -> Result<(String, Option)> { +fn handle_request(config: &NetdConfig, request: Request) -> Result { + let libvirt_uri = config.libvirt_uri.as_str(); let _lock = OperationLock::acquire()?; match request { Request::PrepareBridge(request) => { - prepare_bridge(libvirt_uri, &request).map(|tap| (tap, None)) + prepare_bridge(libvirt_uri, &request, config.filter_policy()) } - Request::PrepareMacvtap(request) => prepare_macvtap( - libvirt_uri, - &request.identity, - &request.parent, - &request.mac, - request.qemu_uid, - &request.mode, - ) - .map(|(tap, device)| (tap, Some(device))), - Request::Remove { identity } => { + Request::PrepareMacvtap(request) => { + prepare_macvtap(libvirt_uri, &request, config.filter_policy()) + } + Request::Remove { identity, filtered } => { validate_identity(&identity)?; let tap = tap_name(&identity); - remove_interface(libvirt_uri, &tap)?; - Ok((tap, None)) + remove_interface(libvirt_uri, &tap, binding_cleanup(filtered))?; + Ok(Prepared::tap(tap)) } - Request::Check { identity } => { + Request::Check { identity, filtered } => { validate_identity(&identity)?; let tap = tap_name(&identity); if !Path::new("/sys/class/net").join(&tap).exists() { bail!("TAP {tap} does not exist"); } - if !is_macvtap(&tap) { + // An unfiltered TAP has no binding to dump; asking for one would + // report a healthy multiqueue interface as broken. + if filtered && !is_macvtap(&tap) { virsh(libvirt_uri, &["nwfilter-binding-dumpxml", &tap], None)?; } - Ok((tap, None)) + Ok(Prepared::tap(tap)) } } } fn prepare_macvtap( libvirt_uri: &str, - identity: &InterfaceIdentity, - parent: &str, - mac: &str, - qemu_uid: u32, - mode: &str, -) -> Result<(String, String)> { + request: &PrepareMacvtapRequest, + filter: &NetworkFilterConfig, +) -> Result { + let identity = &request.identity; + let parent = request.parent.as_str(); + let qemu_uid = request.qemu_uid; validate_identity(identity)?; validate_name("parent", parent, 15, "_.-")?; if !Path::new("/sys/class/net").join(parent).exists() { bail!("parent interface {parent} does not exist"); } - validate_mac(mac)?; - let mode = if mode.is_empty() { "private" } else { mode }; + // A macvtap parent may be a bridge, and libvirt nwfilter does not apply to + // macvtap. So on a node that requires every bridge TAP to be filtered, a + // macvtap request naming that same bridge is the identical unfiltered L2 + // access the policy exists to refuse, spelled with a different operation. + // An interface enslaved to a bridge reaches the same segment. + if filter.requires_binding() { + let sysfs = Path::new("/sys/class/net").join(parent); + if sysfs.join("bridge").exists() { + bail!("this netd requires filtering, so {parent} may not be a macvtap parent: it is a host bridge"); + } + if sysfs.join("master").exists() { + bail!("this netd requires filtering, so {parent} may not be a macvtap parent: it is enslaved to a bridge"); + } + } + validate_mac(&request.mac)?; + let mac = request.mac.as_str(); + let mode = if request.mode.is_empty() { + "private" + } else { + request.mode.as_str() + }; if !matches!(mode, "private" | "bridge" | "vepa" | "passthru") { bail!("invalid macvtap mode"); } + let queues = validate_queues(request.queues)?; let tap = tap_name(identity); - remove_interface(libvirt_uri, &tap)?; - ip(&[ - "link", "add", "link", parent, "name", &tap, "address", mac, "type", "macvtap", "mode", - mode, - ])?; + remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; + let queue_count = queues.to_string(); + let mut add = vec!["link", "add", "link", parent, "name", &tap, "address", mac]; + if queues > 1 { + // macvtap defaults to a single hardware queue pair. Without this the + // extra tap queues exist but the lower device still serializes. + add.extend_from_slice(&["numtxqueues", &queue_count, "numrxqueues", &queue_count]); + } + add.extend_from_slice(&["type", "macvtap", "mode", mode]); + ip(&add)?; let result = (|| { let ifindex = std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifindex")) @@ -346,11 +454,15 @@ fn prepare_macvtap( })(); match result { Ok(device) => { - info!(%tap, %parent, %mode, %device, "prepared macvtap"); - Ok((tap, device)) + info!(%tap, %parent, %mode, %device, %queues, "prepared macvtap"); + Ok(Prepared { + tap, + device: Some(device), + queues: Some(queues), + }) } Err(error) => { - let _ = remove_interface(libvirt_uri, &tap); + let _ = remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort); Err(error) } } @@ -384,41 +496,87 @@ impl Drop for OperationLock { } } -fn prepare_bridge(libvirt_uri: &str, request: &PrepareBridgeRequest) -> Result { - validate_prepare_bridge(request)?; +fn prepare_bridge( + libvirt_uri: &str, + request: &PrepareBridgeRequest, + filter: &NetworkFilterConfig, +) -> Result { + validate_prepare_bridge(request, filter)?; + let filtered = request.filtered; let tap = tap_name(&request.identity); // A failed VMM start may leave a deterministic resource behind. Replacing // it makes prepare idempotent without accepting a caller-selected TAP. - remove_interface(libvirt_uri, &tap)?; + remove_interface(libvirt_uri, &tap, binding_cleanup(filtered))?; let uid = request.qemu_uid.to_string(); - ip(&["tuntap", "add", "dev", &tap, "mode", "tap", "user", &uid])?; + let queues = validate_queues(request.queues)?; + let mut add = vec!["tuntap", "add", "dev", &tap, "mode", "tap"]; + if queues > 1 { + // QEMU refuses to attach when the device's IFF_MULTI_QUEUE state does + // not match its own `queues=` argument, in either direction. + add.push("multi_queue"); + } + add.extend_from_slice(&["user", &uid]); + ip(&add)?; let result = (|| { ip(&["link", "set", "dev", &tap, "master", &request.bridge])?; - let xml = binding_xml(request, &tap); - virsh( - libvirt_uri, - &["nwfilter-binding-create", "--validate", "/dev/stdin"], - Some(xml.as_bytes()), - )?; + if filtered { + let xml = binding_xml(request, &tap, filter); + virsh( + libvirt_uri, + &["nwfilter-binding-create", "--validate", "/dev/stdin"], + Some(xml.as_bytes()), + )?; + } ip(&["link", "set", "dev", &tap, "up"])?; Ok(()) })(); if let Err(error) = result { - let _ = remove_interface(libvirt_uri, &tap); + let _ = remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort); return Err(error); } - info!(%tap, bridge = %request.bridge, filter = %request.filter, "prepared filtered TAP"); - Ok(tap) + info!(%tap, bridge = %request.bridge, %filtered, %queues, "prepared TAP"); + Ok(Prepared { + tap, + device: None, + queues: Some(queues), + }) } -fn remove_interface(libvirt_uri: &str, tap: &str) -> Result<()> { +/// How hard removal must try to clear an nwfilter binding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BindingCleanup { + /// The binding must be gone before this returns, because the caller is + /// about to create one at the same interface name and libvirt refuses a + /// duplicate. + Required, + /// Delete a binding if libvirt can be reached, but do not fail the removal + /// when it cannot. Unfiltered TAPs live on nodes where `libvirtd` need not + /// be running at all, and a stale binding left by an earlier, filtered + /// interface at this name is still worth clearing when it is. + BestEffort, +} + +fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Result<()> { let macvtap = is_macvtap(tap); if Path::new("/sys/class/net").join(tap).exists() { let _ = ip(&["link", "set", "dev", tap, "down"]); } + // A macvtap interface never carries a binding. Anything else might: this + // name may have been a filtered bridge TAP before, and the binding + // outlives the interface. if !macvtap { - delete_binding(libvirt_uri, tap)?; + match cleanup { + BindingCleanup::Required => delete_binding(libvirt_uri, tap)?, + BindingCleanup::BestEffort => { + // netd refuses to start without virsh, so the binary is always + // here; libvirtd need not be running, and on a node that only + // wants macvtap or multiqueue it usually is not. + if let Err(error) = delete_binding(libvirt_uri, tap) { + warn!(%tap, "could not clear a possible nwfilter binding: {error:#}"); + } + } + } } if Path::new("/sys/class/net").join(tap).exists() { ip(&["link", "delete", "dev", tap])?; @@ -434,32 +592,36 @@ fn is_macvtap(interface: &str) -> bool { .exists() } +/// Deletes an interface's nwfilter binding, if it has one. +/// +/// Goes through the same `COMMAND_TIMEOUT`-bounded helper as every other virsh +/// call. netd's accept loop is strictly serialized, so an unbounded call here +/// would let one unreachable libvirt stall every other VM's prepare and remove. fn delete_binding(uri: &str, tap: &str) -> Result<()> { - let output = Command::new(VIRSH_PATH) - .args(["--connect", uri, "nwfilter-binding-delete", tap]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .context("failed to execute virsh")?; - if output.status.success() { - return Ok(()); - } - let error = String::from_utf8_lossy(&output.stderr); - if error.contains("Network filter binding not found") { - return Ok(()); + match virsh(uri, &["nwfilter-binding-delete", tap], None) { + Ok(()) => Ok(()), + // Removal is idempotent. Having no binding is the normal case for + // macvtap, for unfiltered multiqueue TAPs, and for any name being + // reused after an earlier removal already cleared it. + Err(error) + if error + .to_string() + .contains("Network filter binding not found") => + { + Ok(()) + } + Err(error) => Err(error).context(format!("virsh failed to delete binding {tap}")), } - bail!("virsh failed to delete binding {tap}: {}", error.trim()) } -fn binding_xml(request: &PrepareBridgeRequest, tap: &str) -> String { +fn binding_xml(request: &PrepareBridgeRequest, tap: &str, filter: &NetworkFilterConfig) -> String { let owner_uuid = stable_uuid(&request.identity); let owner_name = format!( "dstack:{}:{}:{}", request.identity.instance_id, request.identity.vm_id, request.identity.nic_index ); let mut parameters = String::new(); - for (name, value) in &request.parameters { + for (name, value) in &filter.parameters { parameters.push_str(&format!( "", xml_escape(name), @@ -474,7 +636,7 @@ fn binding_xml(request: &PrepareBridgeRequest, tap: &str) -> String { owner_uuid, xml_escape(tap), xml_escape(&request.mac), - xml_escape(&request.filter), + xml_escape(&filter.filter), parameters ) } @@ -494,9 +656,24 @@ fn stable_uuid(identity: &InterfaceIdentity) -> Uuid { Uuid::from_bytes(bytes) } -fn validate_prepare_bridge(request: &PrepareBridgeRequest) -> Result<()> { +fn validate_prepare_bridge( + request: &PrepareBridgeRequest, + filter: &NetworkFilterConfig, +) -> Result<()> { validate_identity(&request.identity)?; validate_name("bridge", &request.bridge, 15, "_.-")?; + // Unfiltered bridge TAPs exist for unfiltered multiqueue, and netd holds + // that policy itself rather than trusting the caller with it. netd is the + // privileged side of this socket; on a node configured to filter bridge + // traffic, "build me a TAP on br0 with no nwfilter binding" is precisely + // the request the boundary exists to refuse, and anything that can reach + // the socket can make it. + // + // Refused before the host is inspected: this is about the request, not + // about what happens to exist on this machine. + if filter.requires_binding() && !request.filtered { + bail!("this netd requires an nwfilter binding on every bridge TAP"); + } if !Path::new("/sys/class/net") .join(&request.bridge) .join("bridge") @@ -505,16 +682,6 @@ fn validate_prepare_bridge(request: &PrepareBridgeRequest) -> Result<()> { bail!("{} is not a host bridge", request.bridge); } validate_mac(&request.mac)?; - validate_name("filter", &request.filter, 128, "_.:-")?; - if request.parameters.len() > 64 { - bail!("too many nwfilter parameters"); - } - for (name, value) in &request.parameters { - validate_name("parameter name", name, 64, "_")?; - if value.len() > 512 || value.contains('\0') { - bail!("invalid nwfilter parameter value"); - } - } Ok(()) } @@ -533,9 +700,34 @@ fn validate_identity(identity: &InterfaceIdentity) -> Result<()> { Ok(()) } +/// A caller that knows a binding is there needs it gone; one that does not +/// still clears whatever it finds, without failing when libvirt is absent. +fn binding_cleanup(filtered: bool) -> BindingCleanup { + if filtered { + BindingCleanup::Required + } else { + BindingCleanup::BestEffort + } +} + +/// Normalizes a requested queue pair count. Zero means the caller did not ask +/// for multiqueue, which is the same device shape as one queue pair. +fn validate_queues(queues: u32) -> Result { + if queues > MAX_QUEUES { + bail!("queues must not exceed {MAX_QUEUES}"); + } + Ok(queues.max(1)) +} + fn validate_name(label: &str, value: &str, max: usize, punctuation: &str) -> Result<()> { if value.is_empty() || value.len() > max + // `.` and `..` pass the charset check below, and every name validated + // here is then joined onto a sysfs path to ask whether the interface + // exists. `/sys/class/net/..` exists, so the question would be answered + // about a directory rather than about an interface. + || value == "." + || value == ".." || !value .chars() .all(|ch| ch.is_ascii_alphanumeric() || punctuation.contains(ch)) @@ -648,6 +840,7 @@ fn prepare_socket_path(socket: &Path) -> Result<()> { #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeMap; fn identity(instance: &str, vm: &str, nic_index: usize) -> InterfaceIdentity { InterfaceIdentity { @@ -673,10 +866,15 @@ mod tests { bridge: "br0".into(), mac: "02:00:00:00:00:01".into(), qemu_uid: 1000, + filtered: true, + queues: 0, + }; + let filter = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, filter: "clean-traffic".into(), parameters: BTreeMap::from([("IP".into(), "10.0.0.2<&".into())]), }; - let xml = binding_xml(&request, "dt123"); + let xml = binding_xml(&request, "dt123", &filter); assert!(xml.contains("instance<&")); assert!(xml.contains("10.0.0.2<&")); assert!(!xml.contains("instance<&")); @@ -685,6 +883,10 @@ mod tests { #[test] fn validation_rejects_injected_host_names() { assert!(validate_name("bridge", "br0;id", 15, "_.-").is_err()); + // `/sys/class/net/..` exists, so an existence check on this name would + // answer about a directory rather than about an interface. + assert!(validate_name("parent", "..", 15, "_.-").is_err()); + assert!(validate_name("parent", ".", 15, "_.-").is_err()); assert!(validate_name("filter", "../../filter", 128, "_.:-").is_err()); assert!(validate_mac("ff:ff:ff:ff:ff:ff").is_err()); } @@ -693,6 +895,7 @@ mod tests { fn remove_protocol_keeps_identity_fields_flat() { let request = Request::Remove { identity: identity("instance", "vm", 2), + filtered: true, }; let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "remove"); @@ -709,8 +912,8 @@ mod tests { bridge: "br0".into(), mac: "02:00:00:00:00:01".into(), qemu_uid: 1000, - filter: "clean-traffic".into(), - parameters: BTreeMap::new(), + filtered: true, + queues: 0, }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -719,6 +922,90 @@ mod tests { assert!(value.get("identity").is_none()); } + /// `filtered` says which of two shapes was built, and both are reachable + /// on any node this build can produce. There is no released peer that omits + /// it -- netd does not exist before v0.6 -- so it is required rather than + /// defaulted, and a request that leaves it out is a bug, not an old client. + #[test] + fn removal_states_which_shape_it_is_undoing() { + let error = serde_json::from_value::(serde_json::json!({ + "operation": "remove", + "instance_id": "instance", + "vm_id": "vm", + "nic_index": 0, + })) + .unwrap_err(); + assert!(error.to_string().contains("filtered"), "{error}"); + + for filtered in [true, false] { + let decoded: Request = serde_json::from_value(serde_json::json!({ + "operation": "remove", + "instance_id": "instance", + "vm_id": "vm", + "nic_index": 0, + "filtered": filtered, + })) + .unwrap(); + let Request::Remove { + filtered: decoded, .. + } = decoded + else { + panic!("wrong variant"); + }; + assert_eq!(decoded, filtered); + } + } + + /// A binding outlives the interface it was bound to, and TAP names are a + /// deterministic hash of the VM identity, so the same name comes back. + /// Removing an interface therefore clears whatever binding is there, and + /// only insists when the caller is about to create a replacement. + #[test] + fn binding_cleanup_insists_only_when_a_replacement_follows() { + assert_eq!(binding_cleanup(true), BindingCleanup::Required); + assert_eq!(binding_cleanup(false), BindingCleanup::BestEffort); + } + + #[test] + fn queue_counts_normalize_to_at_least_one_and_stay_bounded() { + assert_eq!(validate_queues(0).unwrap(), 1); + assert_eq!(validate_queues(1).unwrap(), 1); + assert_eq!(validate_queues(MAX_QUEUES).unwrap(), MAX_QUEUES); + assert!(validate_queues(MAX_QUEUES + 1).is_err()); + } + + #[test] + fn queue_count_travels_with_the_prepare_request() { + let request = Request::PrepareBridge(PrepareBridgeRequest { + identity: identity("instance", "vm", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: false, + queues: 4, + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["queues"], 4); + assert_eq!(value["filtered"], false); + + // QEMU refuses a device whose IFF_MULTI_QUEUE state disagrees with its + // own `queues=`, so a request that leaves the count to netd's + // imagination is one netd must not answer. + let error = serde_json::from_value::(serde_json::json!({ + "operation": "prepare_bridge", + "instance_id": "instance", + "vm_id": "vm", + "nic_index": 0, + "bridge": "br0", + "mac": "02:00:00:00:00:01", + "qemu_uid": 1000, + "filtered": true, + })) + .unwrap_err(); + assert!(error.to_string().contains("queues"), "{error}"); + assert_eq!(validate_queues(0).unwrap(), 1); + } + #[test] fn macvtap_prepare_has_a_dedicated_operation() { let request = Request::PrepareMacvtap(PrepareMacvtapRequest { @@ -727,6 +1014,7 @@ mod tests { mac: "02:00:00:00:00:01".into(), qemu_uid: 1000, mode: "private".into(), + queues: 0, }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_macvtap"); @@ -742,8 +1030,8 @@ mod tests { bridge: "br0".into(), mac: "02:00:00:00:00:01".into(), qemu_uid: 1000, - filter: "clean-traffic".into(), - parameters: BTreeMap::new(), + filtered: true, + queues: 0, }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -752,8 +1040,14 @@ mod tests { assert!(value.get("identity").is_none()); } + /// Connecting and closing without sending is how the VMM checks that netd + /// is alive, because a netd that died leaves its socket behind. It has to + /// be handled promptly, and quietly: the VMM does it once per status query + /// that mentions a stopped VM, and netd's accept loop is serialized, so + /// treating a probe as a failed request would both fill the log and put + /// noise in front of real work. #[tokio::test] - async fn disconnected_client_is_confined_to_one_connection() { + async fn a_connection_that_sends_nothing_is_a_liveness_probe() { let (mut server, client) = UnixStream::pair().unwrap(); drop(client); let result = timeout( @@ -762,10 +1056,143 @@ mod tests { ) .await; assert!(result.is_ok(), "disconnected peer blocked the handler"); - // Either the EOF is reported while reading or the response write sees - // EPIPE. In both cases serve() logs this per-connection error and keeps - // accepting clients. - assert!(result.unwrap().is_err()); + assert!(result.unwrap().is_ok(), "a probe is not a failed request"); + } + + /// Only an empty connection is a probe. A peer that does send something, + /// and sends nonsense, is still a request -- and still gets an answer it + /// can read, which is how a VMM newer than its netd learns to say so. + #[tokio::test] + async fn a_request_that_cannot_be_understood_still_gets_an_answer() { + let (mut server, client) = UnixStream::pair().unwrap(); + drop(client); + assert!(read_request(&mut server).await.unwrap().is_none()); + + let (mut server, mut client) = UnixStream::pair().unwrap(); + // An operation only a newer VMM knows about. + client + .write_all(br#"{"operation":"prepare_something_new"}"#) + .await + .unwrap(); + client.shutdown().await.unwrap(); + serve_connection(&NetdConfig::default(), &mut server) + .await + .unwrap(); + + let mut reply = Vec::new(); + client.read_to_end(&mut reply).await.unwrap(); + let reply: serde_json::Value = serde_json::from_slice(&reply).unwrap(); + assert_eq!(reply["ok"], false); + assert!( + reply["error"].as_str().unwrap().contains("unknown variant"), + "{reply}" + ); + } + + /// netd is the privileged side of this socket. "Build me a TAP on br0 with + /// no nwfilter binding" is the request the boundary exists to refuse on a + /// filtering node, and before this the daemon simply did what it was told, + /// leaving the invariant with the unprivileged caller. + #[test] + fn a_filtering_node_refuses_an_unfiltered_bridge_tap() { + let request = PrepareBridgeRequest { + identity: InterfaceIdentity { + instance_id: "i".into(), + vm_id: "v".into(), + nic_index: 0, + }, + // A name no host has, so the check after this one is the one that + // fails when this one does not. + bridge: "dstack-nobr0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: false, + queues: 4, + }; + let filtering = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, + ..NetworkFilterConfig::default() + }; + let error = validate_prepare_bridge(&request, &filtering).unwrap_err(); + assert!( + error.to_string().contains("requires an nwfilter binding"), + "{error}" + ); + + // An unfiltered node still builds them; that is what multiqueue needs. + // It gets as far as asking the host about the bridge, which is the + // next check and not this one's business. + let error = validate_prepare_bridge(&request, &NetworkFilterConfig::default()).unwrap_err(); + assert!( + error.to_string().contains("is not a host bridge"), + "{error}" + ); + } + + /// nwfilter does not apply to macvtap, and a macvtap parent may be the very + /// bridge the policy protects. Refusing an unfiltered bridge TAP while + /// handing out a macvtap on the same segment would leave the policy + /// enforced only against the spelling that happens to be checked. + #[test] + fn a_filtering_node_refuses_a_macvtap_parent_that_is_a_bridge() { + let filtering = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, + ..NetworkFilterConfig::default() + }; + let bridges: Vec = std::fs::read_dir("/sys/class/net") + .into_iter() + .flatten() + .flatten() + .filter(|entry| entry.path().join("bridge").exists()) + .filter_map(|entry| entry.file_name().into_string().ok()) + .collect(); + let Some(bridge) = bridges.first() else { + // Nothing to assert against on a host with no bridge; the unit + // below still pins the enslaved case's sysfs predicate. + return; + }; + let request = PrepareMacvtapRequest { + identity: identity("i", "v", 0), + parent: bridge.clone(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + mode: "bridge".into(), + queues: 4, + }; + let error = match prepare_macvtap("test:///default", &request, &filtering) { + Err(error) => error, + Ok(_) => panic!("a filtering node must not build a macvtap on a host bridge"), + }; + assert!(error.to_string().contains("is a host bridge"), "{error}"); + } + + /// The request says whether to bind a filter, never which one. `allow-arp` + /// contains no drop rule at all, and `clean-traffic` pinned to the + /// gateway's MAC and IP through its parameters filters nothing useful + /// either -- both would satisfy a policy that only asked for "some filter". + #[test] + fn the_bound_filter_comes_from_netds_own_configuration() { + let request = PrepareBridgeRequest { + identity: identity("i", "v", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: true, + queues: 1, + }; + // Nothing on the wire can name a filter: the field does not exist. + let wire = serde_json::to_value(Request::PrepareBridge(request.clone())).unwrap(); + assert!(wire.get("filter").is_none(), "{wire}"); + assert!(wire.get("parameters").is_none(), "{wire}"); + + let policy = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, + filter: "clean-traffic".into(), + parameters: BTreeMap::from([("IP".into(), "10.0.0.2".into())]), + }; + let xml = binding_xml(&request, "dt123", &policy); + assert!(xml.contains("filter='clean-traffic'"), "{xml}"); + assert!(xml.contains("value='10.0.0.2'"), "{xml}"); } #[test] diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index dd7a23d79..d7500b008 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -3,10 +3,11 @@ // SPDX-License-Identifier: Apache-2.0 use crate::app::{ - make_sys_config, resolved_networks, simulator_config_for_manifest, sync_tee_simulator_config, - Image, VmConfig, VmWorkDir, + clamp_queues_without_netd, make_sys_config, needs_netd_interface, resolved_networks, + settle_vhost, simulator_config_for_manifest, sync_tee_simulator_config, Image, VmConfig, + VmWorkDir, }; -use crate::config::{Config, NetworkFilterMode, NetworkingMode}; +use crate::config::Config; use crate::main_service; use anyhow::{Context, Result}; use fs_err as fs; @@ -279,18 +280,45 @@ Compose file content (first 200 chars): gateway_enabled: app_compose.gateway_enabled(), }; + // One-shot has no netd lifecycle, so a bridge NIC that only wanted the + // vCPU-scaled default drops to a single queue here exactly as it would on a + // server without netd. Anything still needing an interface was asked for + // explicitly, and is refused rather than silently downgraded. + let requested = if manifest.networks.is_empty() { + vec![config.cvm.networking.nic.clone()] + } else { + manifest.networks.clone() + }; + let mut runtime_networks = resolved_networks(&manifest, &config.cvm); + let clamped = clamp_queues_without_netd(&requested, &mut runtime_networks, &config.cvm, false); + // The server settles vhost after clamping, because clamping changes whether + // a NIC needs netd and that changes which netdev it gets. Skipping it here + // left `vhost_enabled()` reading as a request rather than a decision, so + // the launch warned about a `/dev/vhost-net` the netdev it then built does + // not open. + let vhost_denied = settle_vhost(&mut runtime_networks, &config.cvm); + if vhost_denied > 0 { + tracing::warn!( + "no qemu-bridge-helper found, so {vhost_denied} bridge interface(s) fall back to the \ + non-vhost bridge netdev; set cvm.qemu_bridge_helper to enable vhost" + ); + } + if clamped > 0 { + tracing::warn!( + "one-shot execution has no netd, so {clamped} bridge interface(s) fall back to a \ + single queue pair; run the VMM server to let queue pairs scale with vCPUs" + ); + } if !dry_run - && config.cvm.network_filter.mode == NetworkFilterMode::Libvirt - && resolved_networks(&manifest, &config.cvm) + && runtime_networks .iter() - .any(|network| network.mode == NetworkingMode::Bridge) + .any(|network| needs_netd_interface(network, &config.cvm)) { anyhow::bail!( - "one-shot execution does not manage libvirt-filtered TAP lifecycle; run the VMM server directly or use --dry-run" + "one-shot execution does not manage netd interface lifecycle; run the VMM server directly or use --dry-run" ); } - let runtime_networks = resolved_networks(&manifest, &config.cvm); let process_configs = vm_builder_config .config_qemu(&workdir_path, &config.cvm, &gpus, &runtime_networks) .context("Failed to build QEMU configuration")?; diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 1a533434f..20e515520 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -919,8 +919,18 @@ def create_vm(self, args) -> None: params["kms_urls"] = args.kms_url if args.gateway_url: params["gateway_urls"] = args.gateway_url - if args.net: - params["networking"] = {"mode": args.net} + # "auto" is what a fresh deployment already does, so it only means + # something to `update`, where it clears a pinned count. + net_queues = None if args.net_queues == "auto" else args.net_queues + if args.net or args.net_vhost is not None or net_queues: + networking = {} + if args.net: + networking["mode"] = args.net + if args.net_vhost is not None: + networking["vhost"] = args.net_vhost + if net_queues: + networking["queues"] = net_queues + params["networking"] = networking app_id = args.app_id or self.calc_app_id(compose_content) print(f"App ID: {app_id}") @@ -1030,6 +1040,10 @@ def update_vm( no_gpus: bool = False, kms_urls: Optional[List[str]] = None, no_tee: Optional[bool] = None, + net: Optional[str] = None, + net_vhost: Optional[bool] = None, + net_vhost_inherit: bool = False, + net_queues: Optional[Union[int, str]] = None, ) -> None: """Update multiple aspects of a VM in one command.""" # Validate: --env-file requires --kms-url @@ -1153,6 +1167,75 @@ def update_vm( app_compose, indent=4, ensure_ascii=False ) + if net or net_vhost is not None or net_vhost_inherit or net_queues: + # The RPC replaces the whole NIC list, so merge into what the VM + # already has rather than silently dropping its other interfaces or + # un-pinning a bridge it was deployed with. + if vm_info_response is None: + vm_info_response = self.rpc_call("GetInfo", {"id": vm_id}) + if not vm_info_response.get("found", False): + raise Exception(f"VM with ID {vm_id} not found") + configuration = vm_info_response["info"].get("configuration") or {} + current = configuration.get("networks") or [] + if not current and configuration.get("networking"): + current = [configuration["networking"]] + if len(current) > 1: + raise Exception( + "this VM has multiple network interfaces; edit them through the " + "web UI or the UpgradeApp API rather than these flags" + ) + # Only the fields the deployment RPC accepts back travel with the + # update. macvtap_mode is node-controlled and can never be changed, + # so resending it can only fail if the node changed meanwhile. An + # empty mode is meaningful: it says the VM never named a backend + # and still follows the node's. + source = current[0] if current else {} + networking = { + key: source[key] + for key in ("mode", "bridge_name", "parent", "vhost", "queues") + if source.get(key) not in (None, "") + } + if net == "default": + # The only way back to "whatever backend the node runs". Without + # it a VM that named a mode once is pinned to it for life, since + # the merge above carries the reported mode forward on every + # later update. The data plane keeps whatever it was told. + networking.pop("mode", None) + networking.pop("bridge_name", None) + networking.pop("parent", None) + elif net: + networking["mode"] = net + # A field belongs to the mode that owns it. Carrying a bridge into + # a macvtap request, or a parent into a bridge one, asks the server + # about a field the caller never typed and has no flag to clear. + mode = networking.get("mode", "") + if mode and mode != "bridge": + networking.pop("bridge_name", None) + if mode and mode != "macvtap": + networking.pop("parent", None) + # Same rule for the data plane. User networking has neither a vhost + # backend nor multiple queues, so carrying an inherited pin into it + # is rejected for a flag the operator never typed -- and the two + # flags that would clear it are the ones they have not found yet. + # An explicitly typed value still earns the error: that one is + # theirs to be wrong about. + if mode == "user": + if net_vhost is None: + networking.pop("vhost", None) + if not net_queues: + networking.pop("queues", None) + if net_vhost is not None: + networking["vhost"] = net_vhost + elif net_vhost_inherit: + networking.pop("vhost", None) + if net_queues == "auto": + networking.pop("queues", None) + elif net_queues: + networking["queues"] = net_queues + upgrade_params["update_networking"] = True + upgrade_params["networks"] = [networking] + updates.append(f"networking ({networking})") + if user_config: upgrade_params["user_config"] = user_config updates.append("user config") @@ -1247,6 +1330,38 @@ def show_info(self, vm_id: str, json_output: bool = False) -> None: if info.get("shutdown_progress"): print(f"Shutdown: {info['shutdown_progress']}") + interfaces = info.get("interfaces") or [] + if interfaces: + print("\nNetwork Interfaces:") + for iface in interfaces: + parts = [ + f"{iface.get('netdev_id') or '-':<6}", + f"{iface.get('mode') or '-'}/{iface.get('backend') or '-'}", + iface.get("mac") or "-", + ] + if iface.get("bridge_name"): + parts.append(f"bridge={iface['bridge_name']}") + if iface.get("macvtap_mode"): + parts.append(f"macvtap_mode={iface['macvtap_mode']}") + # Absent, not false: custom mode carries an operator-written + # netdev string the VMM never parses, so it reports no data + # plane rather than asserting the resolved default over one that + # may well say vhost=on,queues=8. + vhost = iface.get("vhost") + parts.append( + "vhost=" + ("-" if vhost is None else ("on" if vhost else "off")) + ) + parts.append(f"queues={iface.get('queues') or '-'}") + print(" " + " ".join(parts)) + # A stopped VM has no interfaces to describe, so these are what its + # next launch would build -- which can differ from its last one. + # + # The server's own predicate, not the status string: a VM being + # removed with QEMU still up reports the interfaces that process + # built, and no status value says so. + if not info.get("running", False): + print(" (not running; shown as its next launch would build them)") + events = info.get("events", []) if events: print("\nRecent Events:") @@ -1536,6 +1651,26 @@ def save_whitelist(whitelist: List[str]) -> None: json.dump({"trusted_signers": whitelist}, f, indent=2) +def queue_count(value: str) -> Union[int, str]: + """Parse a queue pair count the node could act on, or "auto" to stop pinning one. + + Zero would otherwise reach the wire as "unset" and be answered with the + default, and a negative one as a decoding error naming a column offset -- + neither of which tells the caller what they asked for was impossible. + """ + if value == "auto": + return "auto" + try: + count = int(value) + except ValueError: + raise argparse.ArgumentTypeError(f"'{value}' is not a whole number or 'auto'") + if count < 1: + raise argparse.ArgumentTypeError( + f"queue pairs must be at least 1, or 'auto' to follow the vCPU count; got {count}" + ) + return count + + def main(): """Parse arguments and dispatch to the appropriate command handler.""" parser = argparse.ArgumentParser(description="dstack-vmm CLI - Manage VMs") @@ -1831,9 +1966,31 @@ def _patched_format_help(): ) deploy_parser.add_argument( "--net", - choices=["bridge", "user"], + choices=["bridge", "user", "macvtap"], help="Networking mode (default: use global config)", ) + net_vhost = deploy_parser.add_mutually_exclusive_group() + net_vhost.add_argument( + "--net-vhost", + dest="net_vhost", + action="store_true", + default=None, + help="Use the host kernel vhost-net data plane (default: use global config)", + ) + net_vhost.add_argument( + "--net-no-vhost", + dest="net_vhost", + action="store_false", + help="Keep packet processing in the QEMU main loop", + ) + deploy_parser.add_argument( + "--net-queues", + type=queue_count, + metavar="N", + help="virtio-net queue pairs, bounded by the node's max_net_queues. " + "Without --net, the node's own networking mode is kept " + "(default: use global config)", + ) # Images command lsimage_parser = subparsers.add_parser("lsimage", help="List available images") @@ -1934,6 +2091,42 @@ def _patched_format_help(): "--env-file", help="File with environment variables to encrypt" ) update_parser.add_argument("--user-config", help="Path to user config file") + update_parser.add_argument( + "--net", + choices=["bridge", "user", "macvtap", "default"], + help=( + "Networking mode (applies from the next boot). 'default' stops " + "pinning a mode and follows the node's, the way --net-queues auto " + "and --net-vhost-default stop pinning the data plane" + ), + ) + update_net_vhost = update_parser.add_mutually_exclusive_group() + update_net_vhost.add_argument( + "--net-vhost", + dest="net_vhost", + action="store_true", + default=None, + help="Use the host kernel vhost-net data plane", + ) + update_net_vhost.add_argument( + "--net-no-vhost", + dest="net_vhost", + action="store_false", + help="Keep packet processing in the QEMU main loop", + ) + update_net_vhost.add_argument( + "--net-vhost-default", + dest="net_vhost_inherit", + action="store_true", + help="Stop pinning vhost and follow the node default again", + ) + update_parser.add_argument( + "--net-queues", + type=queue_count, + metavar="N", + help="virtio-net queue pairs, bounded by the node's max_net_queues. " + "Use 'auto' to stop pinning a count and follow the vCPU count again", + ) # Port mapping options (mutually exclusive with --no-ports) port_group = update_parser.add_mutually_exclusive_group() port_group.add_argument( @@ -2077,6 +2270,10 @@ def _patched_format_help(): no_gpus=args.no_gpus if hasattr(args, "no_gpus") else False, kms_urls=args.kms_url, no_tee=args.no_tee, + net=args.net, + net_vhost=args.net_vhost, + net_vhost_inherit=getattr(args, "net_vhost_inherit", False), + net_queues=args.net_queues, ) elif args.command == "kms": if not args.kms_action: diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index 82653a62d..27adcaecd 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -22,7 +22,44 @@ const CreateVmDialogComponent = { portMappingEnabled: { type: Boolean, required: true }, networkingModes: { type: Array, required: true }, defaultBridge: { type: String, default: '' }, + maxNetQueues: { type: Number, default: 0 }, defaultNetworkingLabel: { type: String, required: true }, + defaultModeTunable: { type: Boolean, default: false }, + defaultVhostOn: { type: Boolean, default: false }, + }, + methods: { + // Whether this NIC will end up on the vhost data plane. An unset select + // means it follows the node, and a node with vhost off gives one queue pair + // however many vCPUs the VM has -- so the answer is not readable from this + // row alone. + vhostOn(network: { vhost?: string }) { + if (network.vhost === 'on') { + return true; + } + if (network.vhost === 'off') { + return false; + } + return (this as any).defaultVhostOn; + }, + // What an empty queues field actually resolves to. It is the vCPU count + // only when vhost is on: with vhost off the backend has no multiqueue data + // plane and the NIC gets exactly one queue pair. + queuesHint(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'virtio-net queue pairs. Empty means one queue pair, because vhost is off.'; + } + const cap = (this as any).maxNetQueues + ? `, capped at ${(this as any).maxNetQueues} on this node` + : ''; + return `virtio-net queue pairs. Empty follows the VM's vCPU count${cap}.`; + }, + queuesPlaceholder(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'queues: auto (1, vhost off)'; + } + const cap = (this as any).maxNetQueues ? ` (max ${(this as any).maxNetQueues})` : ''; + return `queues: auto${cap}`; + }, }, emits: ['close', 'submit', 'load-compose'], template: /* html */ ` @@ -161,6 +198,7 @@ const CreateVmDialogComponent = {
{{ defaultNetworkingLabel }}
+ + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} Guest IP is assigned by host DHCP on that bridge and reported after boot. + + Leave empty to use the macvtap parent from vmm.toml. The forwarding mode stays node-controlled. + + + {{ defaultNetworkingLabel }} has no vhost-net or multiqueue data plane, so these two settings are recorded + but stay dormant until this node's default backend can carry them. +
- + diff --git a/dstack/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts index 5dc9568b1..ec2bfbfa6 100644 --- a/dstack/vmm/ui/src/components/UpdateVmDialog.ts +++ b/dstack/vmm/ui/src/components/UpdateVmDialog.ts @@ -21,10 +21,47 @@ const UpdateVmDialogComponent = { portMappingEnabled: { type: Boolean, required: true }, networkingModes: { type: Array, required: true }, defaultBridge: { type: String, default: '' }, + maxNetQueues: { type: Number, default: 0 }, defaultNetworkingLabel: { type: String, required: true }, + defaultModeTunable: { type: Boolean, default: false }, + defaultVhostOn: { type: Boolean, default: false }, kmsEnabled: { type: Boolean, required: true }, composeHashPreview: { type: String, required: true }, }, + methods: { + // Whether this NIC will end up on the vhost data plane. An unset select + // means it follows the node, and a node with vhost off gives one queue pair + // however many vCPUs the VM has -- so the answer is not readable from this + // row alone. + vhostOn(network: { vhost?: string }) { + if (network.vhost === 'on') { + return true; + } + if (network.vhost === 'off') { + return false; + } + return (this as any).defaultVhostOn; + }, + // What an empty queues field actually resolves to. It is the vCPU count + // only when vhost is on: with vhost off the backend has no multiqueue data + // plane and the NIC gets exactly one queue pair. + queuesHint(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'virtio-net queue pairs. Empty means one queue pair, because vhost is off.'; + } + const cap = (this as any).maxNetQueues + ? `, capped at ${(this as any).maxNetQueues} on this node` + : ''; + return `virtio-net queue pairs. Empty follows the VM's vCPU count${cap}.`; + }, + queuesPlaceholder(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'queues: auto (1, vhost off)'; + } + const cap = (this as any).maxNetQueues ? ` (max ${(this as any).maxNetQueues})` : ''; + return `queues: auto${cap}`; + }, + }, emits: ['close', 'submit', 'load-compose'], template: /* html */ `
@@ -150,24 +187,67 @@ const UpdateVmDialogComponent = {
{{ defaultNetworkingLabel }}
+ + + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} Guest IP is assigned by host DHCP on that bridge and reported after boot. + + Leave empty to use the macvtap parent from vmm.toml. The forwarding mode stays node-controlled. + + + {{ defaultNetworkingLabel }} has no vhost-net or multiqueue data plane, so these two settings are recorded + but stay dormant until this node's default backend can carry them. +
- +
diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index 1abb5ab75..70e9db9df 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -92,6 +92,7 @@ type VmListItem = { shutdown_progress?: string; image_version?: string; interfaces?: VmmTypes.INetworkInterfaceStatus[]; + running?: boolean; configuration?: VmConfiguration; appCompose?: AppCompose; }; @@ -111,6 +112,15 @@ type PortFormEntry = { type NetworkFormEntry = { mode: string; bridge_name?: string; + /** Pinned at deployment for macvtap NICs; carried through edits unchanged. */ + parent?: string; + /** '' inherits the node default, otherwise 'on' or 'off'. */ + vhost?: string; + /** + * '' lets the queue count follow the vCPU count. A `number` once the operator + * types into the input: `v-model` casts for ``. + */ + queues?: string | number; }; type VmFormState = { @@ -351,6 +361,18 @@ fi return Array.from(new Set(fallback)); }); const defaultBridge = computed(() => config.value.networking?.default_bridge || ''); + const maxNetQueues = computed(() => config.value.networking?.max_queues || 0); + const defaultVhostOn = computed(() => !!config.value.networking?.default_vhost); + // Whether the node's own default backend can carry vhost-net and multiqueue. + // The RPC accepts the two tuning fields on a mode-less entry regardless -- + // deliberately, so a NIC that inherits its backend stays tunable across a node + // change -- but on a node whose default is user or custom they lie dormant, + // and an operator who sets them deserves to be told that rather than discover + // it in the interfaces panel afterwards. + const defaultModeTunable = computed(() => { + const mode = config.value.networking?.default_mode || ''; + return mode === 'bridge' || mode === 'macvtap'; + }); const defaultNetworkingLabel = computed(() => { const mode = config.value.networking?.default_mode || ''; if (mode === 'bridge') { @@ -450,16 +472,67 @@ fi return configured.map((network) => ({ mode: network.mode || '', bridge_name: network.bridge_name || '', + parent: network.parent || '', + vhost: network.vhost === null || network.vhost === undefined ? '' : (network.vhost ? 'on' : 'off'), + queues: network.queues ? String(network.queues) : '', })); }; + // Queue pairs, whatever shape the model is in. + // + // Not a string: Vue's `v-model` casts for ``, so this + // field is a `string` while it holds a value loaded from `GetInfo` and a + // `number` the moment the operator types into it. Assuming either one is how + // this threw a `TypeError` out of every deploy that set a queue count. + // + // The number input already refuses everything but a numeric literal, and the + // cast turns `2.7` into `2.7` rather than into `parseInt`'s `2`, so what is + // left to check is that the value is a whole number at least one. The node's + // cap is deliberately *not* checked here: the server widens it by whatever a + // VM already holds, so a client-side copy would refuse an update the server + // accepts and leave that VM's networking uneditable. + const parseQueueCount = (raw: unknown, label: string): number | undefined => { + if (raw === null || raw === undefined || raw === '') { + return undefined; + } + const queues = typeof raw === 'number' ? raw : Number(String(raw).trim()); + if (!Number.isInteger(queues) || queues < 1) { + throw new Error(`${label}: queue pairs must be a whole number of at least 1, or empty to follow the vCPU count; got '${raw}'`); + } + return queues; + }; + + // Nothing is filtered out. A mode-less entry is the "keep the node's backend, + // change only the data plane" override the RPC accepts, and is what a VM + // deployed that way reports back; dropping it would delete a NIC and renumber + // the ones after it, which changes their MAC addresses. + // A row added here starts with no mode, which the RPC reads as "keep the + // node's backend" rather than as a missing field, so the only way to reach an + // entry it refuses is to empty a loaded one's bridge or parent, and that + // earns an error rather than silence. const normalizeNetworks = (networks: NetworkFormEntry[] = []): VmmTypes.INetworkingConfig[] => networks - .map((network) => ({ - mode: (network.mode || '').trim(), - bridge_name: network.mode === 'bridge' ? (network.bridge_name || '').trim() : '', - })) - .filter((network) => network.mode.length > 0); + .map((network, index) => { + // Leave vhost and queues unset unless the operator picked something, so + // the node keeps owning them and can still change them later. + const entry: VmmTypes.INetworkingConfig = { + mode: (network.mode || '').trim(), + bridge_name: network.mode === 'bridge' ? (network.bridge_name || '').trim() : '', + parent: network.mode === 'macvtap' ? (network.parent || '').trim() : '', + }; + // The tuning controls are hidden for user mode, so sending values the + // operator cannot see would fail the deploy with nothing to fix. + if (network.mode !== 'user') { + if (network.vhost === 'on' || network.vhost === 'off') { + entry.vhost = network.vhost === 'on'; + } + const queues = parseQueueCount(network.queues, `network ${index + 1}`); + if (queues !== undefined) { + entry.queues = queues; + } + } + return entry; + }); function networkModeLabel(mode?: string | null) { if (!mode) { @@ -1014,6 +1087,11 @@ type CreateVmPayloadSource = { function showDeployDialog() { showCreateDialog.value = true; vmForm.value.encryptedEnvs = []; + // A cancelled deploy and "Clone config" both leave their networking behind + // in the shared form. Carrying it into the next deploy would silently pin + // that VM's backend, bridge, macvtap parent and data plane to another VM's + // -- invisibly, since the operator never opened the Networking section. + vmForm.value.networks = []; vmForm.value.app_id = null; vmForm.value.swapValue = 0; vmForm.value.swapUnit = 'GB'; @@ -1833,7 +1911,10 @@ type CreateVmPayloadSource = { config, networkingModes, defaultBridge, + maxNetQueues, defaultNetworkingLabel, + defaultModeTunable, + defaultVhostOn, composeHashPreview, updateComposeHashPreview, showDeployDialog, diff --git a/dstack/vmm/ui/src/styles/main.css b/dstack/vmm/ui/src/styles/main.css index 7a7b0ee72..113109a42 100644 --- a/dstack/vmm/ui/src/styles/main.css +++ b/dstack/vmm/ui/src/styles/main.css @@ -620,7 +620,7 @@ h1, h2, h3, h4, h5, h6 { .runtime-network-item { display: grid; - grid-template-columns: 1.1fr 0.6fr 0.8fr 1.4fr; + grid-template-columns: 1.1fr 0.6fr 0.8fr 1.4fr 0.7fr 0.7fr; gap: 12px; overflow-wrap: anywhere; } @@ -1420,13 +1420,35 @@ h1, h2, h3, h4, h5, h6 { font-size: 14px; } +/* Fixed track widths summed to a floor wider than the dialog, which put a + horizontal scrollbar under the Networking section on any window narrower than + about 930px. The tracks shrink and the tuning pair wraps instead. */ .network-config-row { display: grid; - grid-template-columns: 160px minmax(180px, 1fr) 100px; + grid-template-columns: minmax(120px, 160px) minmax(140px, 1fr) minmax(200px, auto) auto; gap: 12px; align-items: center; } +.network-config-tuning { + display: flex; + flex-wrap: wrap; + gap: 8px; + min-width: 0; +} + +.network-config-tuning select { + flex: 1 1 140px; + min-width: 110px; + max-width: 160px; +} + +.network-config-tuning input { + flex: 1 1 150px; + min-width: 90px; + max-width: 190px; +} + .network-config-placeholder { min-height: 1px; } diff --git a/dstack/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html index 139155032..965f7bcc2 100644 --- a/dstack/vmm/ui/src/templates/app.html +++ b/dstack/vmm/ui/src/templates/app.html @@ -80,7 +80,10 @@

dstack-vmm

:port-mapping-enabled="config.portMappingEnabled" :networking-modes="networkingModes" :default-bridge="defaultBridge" + :max-net-queues="maxNetQueues" :default-networking-label="defaultNetworkingLabel" + :default-mode-tunable="defaultModeTunable" + :default-vhost-on="defaultVhostOn" @close="showCreateDialog = false" @submit="createVm" @load-compose="loadComposeFile" @@ -95,7 +98,10 @@

dstack-vmm

:port-mapping-enabled="config.portMappingEnabled" :networking-modes="networkingModes" :default-bridge="defaultBridge" + :max-net-queues="maxNetQueues" :default-networking-label="defaultNetworkingLabel" + :default-mode-tunable="defaultModeTunable" + :default-vhost-on="defaultVhostOn" :kms-enabled="kmsEnabled(updateDialog.vm || {})" :compose-hash-preview="updateComposeHashPreview" @close="updateDialog.show = false" @@ -345,11 +351,16 @@

Port Mappings

VMM Network Interfaces

+ + Not running; shown as the next launch would build them. +
{{ networkModeLabel(iface.mode) }} / {{ iface.backend || '-' }} {{ iface.netdev_id || '-' }} - {{ iface.bridge_name || '-' }} + {{ iface.bridge_name || iface.macvtap_mode || '-' }} {{ iface.mac || '-' }} + vhost: {{ iface.vhost === null || iface.vhost === undefined ? '-' : (iface.vhost ? 'on' : 'off') }} + queues: {{ iface.queues || '-' }}
diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 26f221866..3a83d5016 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -54,6 +54,16 @@ allowed_network_modes = ["user", "bridge"] # Empty allowlists mean callers can only use the node networking defaults. allowed_bridges = [] allowed_macvtap_parents = [] +# Largest virtio-net queue pair count a deployment request may ask for. With +# vhost on, queue pairs otherwise default to the VM's vCPU count, capped at +# 16 (without vhost the default is a single queue pair); raising this +# above 16 widens what a caller may request without moving that default, and +# lowering it below 16 lowers the default too. Bridge mode needs netd for +# anything above 1, because qemu-bridge-helper cannot create a multiqueue TAP. +# Without netd an unfiltered bridge NIC that took the default drops to one +# queue; one that asked for a count keeps it and fails to launch instead, so +# the caller learns their request was not met. +max_net_queues = 16 use_mrconfigid = true # QEMU flags @@ -62,6 +72,10 @@ use_mrconfigid = true #qemu_version = "" qemu_pci_hole64_size = 0 qemu_hotplug_off = false +# Path to qemu-bridge-helper, needed by vhost bridge networking because QEMU's +# `tap` netdev, unlike its `bridge` netdev, has no compiled-in default. Empty +# probes the known distribution locations. +#qemu_bridge_helper = "/usr/lib/qemu/qemu-bridge-helper" # TDX attestation/hash scheme policy: # - "legacy": digest.txt + legacy verifier # - "lite": digest.txt + measurement.tdx.cbor + no-QEMU verifier @@ -111,6 +125,16 @@ product_name = "dstack" [cvm.networking] mode = "user" +# Kernel vhost-net data plane. Off by default: enabling it changes the +# virtio-net device of every bridge/macvtap VM on its next boot (vhost plus +# vCPU-scaled queue pairs) and requires /dev/vhost-net to be accessible to +# the account QEMU runs under — verify that first, or QEMU exits at launch. +# With it off, QEMU drains every packet on its single main loop thread, so a +# CVM cannot exceed one core's worth of packet processing no matter how many +# vCPUs it has. The user-mode backend has no vhost support and ignores this. +# Individual VMs may override it. +vhost = false + # for mode = "user" net = "10.0.2.0/24" dhcp_start = "10.0.2.10" @@ -120,21 +144,36 @@ restrict = false # bridge = "virbr0" # Optional filtering for bridge interfaces only. It does not apply to macvtap. -# "none" preserves the existing QEMU bridge-helper behavior and has no -# netd/libvirt dependency. +# "none" installs no nwfilter binding. It does not by itself remove the netd +# dependency: netd also builds the multiqueue TAP that qemu-bridge-helper +# cannot create, so a bridge node without netd is limited to one queue pair. [cvm.network_filter] mode = "none" filter = "clean-traffic" parameters = {} -# Shared privileged networking service. Only used when network_filter.mode is -# "libvirt". Socket filesystem permissions authorize clients. +# Shared privileged networking service. Used for macvtap NICs, for libvirt +# filtering, and for multiqueue bridge NICs. Socket filesystem permissions +# authorize clients. [netd] socket = "/run/dstack/netd.sock" # Applied when netd creates the socket itself. A systemd socket unit controls # its own SocketMode instead. socket_mode = 0o660 libvirt_uri = "qemu:///system" +# The bridge filtering policy netd enforces and applies: whether a binding is +# required, which nwfilter it names, and with what parameters. netd holds this +# itself rather than taking it from each request, because a caller that chose +# the filter could name one that drops nothing and still satisfy a policy that +# only asked for "some filter". Left unset it follows [cvm.network_filter] in +# this same file, which is the whole answer when netd and the VMM share one +# vmm.toml. Set it explicitly when netd runs with a config that has no [cvm] +# section, so the daemon holding the privilege never infers policy from a file +# that does not state it. +#[netd.network_filter] +#mode = "libvirt" +#filter = "clean-traffic" +#parameters = {} [cvm.port_mapping] enabled = false From 1aa1d35e61c9cb02e26dd2ae45d4f9e07808f4ff Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 21:20:05 -0700 Subject: [PATCH 2/5] docs: document virtio-net data plane tuning --- docs/bridge-networking.md | 35 ++-- docs/libvirt-network-filter.md | 89 ++++++++-- docs/macvtap-networking.md | 8 +- docs/network-data-plane.md | 306 +++++++++++++++++++++++++++++++++ 4 files changed, 408 insertions(+), 30 deletions(-) create mode 100644 docs/network-data-plane.md diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 0d16fd476..6bf3e6d5e 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -4,7 +4,7 @@ By default, dstack-vmm uses **user** networking (QEMU's built-in SLIRP stack, no ## When to use bridge networking -- High connection concurrency (passt becomes CPU-bound at ~25K+ concurrent connections) +- High connection concurrency (user-mode networking becomes CPU-bound at ~25K+ concurrent connections) - Workloads that need full L2 network access - Environments where VMs need to be directly reachable on the LAN @@ -21,11 +21,11 @@ bridge = "virbr0" ### Per-VM override Individual VMs can override the global networking mode via: -- **CLI**: `vmm-cli.py deploy --net bridge` or `--net passt` +- **CLI**: `vmm-cli.py deploy --net bridge`, `--net user`, or `--net macvtap` - **Web UI**: Networking dropdown in the deploy dialog - **API**: `networking: { mode: "bridge" }` in `VmConfiguration` -Only the mode is per-VM; the bridge interface name always comes from the global config. +The bridge interface name comes from the global config unless the node lists it in `cvm.allowed_bridges`. VMs may also override the vhost and queue settings — see [network-data-plane.md](network-data-plane.md). ## Host setup @@ -143,9 +143,15 @@ mode = "bridge" bridge = "dstack-br0" ``` -### QEMU bridge helper setup (required for both options) +### QEMU bridge helper setup (needed unless every bridge NIC goes through netd) The bridge helper allows QEMU to create and attach TAP devices without VMM needing root privileges. +It is used only on the single-queue bridge paths; a NIC that `netd` builds never touches it, so a +node that runs `netd` for all of its bridge VMs does not need it at all. + +The VMM probes `/usr/lib/qemu/qemu-bridge-helper`, `/usr/libexec/qemu-bridge-helper` and +`/usr/local/libexec/qemu-bridge-helper`. Set `cvm.qemu_bridge_helper` in `vmm.toml` for a path +outside that list. ```bash # Allow QEMU to use the bridge @@ -159,12 +165,13 @@ sudo chmod u+s /usr/lib/qemu/qemu-bridge-helper ## How it works -- VMM passes `-netdev bridge,id=net0,br=` to QEMU -- QEMU's bridge helper (setuid) creates a TAP device and attaches it to the bridge +- With more than one queue pair, or with libvirt filtering on, `netd` creates the TAP and the VMM passes `-netdev tap,id=net0,ifname=,...` — this is the usual case on a node running `netd` with multi-vCPU VMs, since queue pairs default to the VM's vCPU count. Without `netd`, a bridge NIC that took that default drops back to one queue pair and takes a helper path below +- Otherwise the VMM passes `-netdev tap,id=net0,br=,helper=,vhost=on`, or `-netdev bridge,id=net0,br=` when vhost is off or no helper is found +- QEMU's bridge helper (setuid) creates a TAP device and attaches it to the bridge on the two helper paths - Guest MAC address is derived from SHA256 of the VM ID, with an optional configurable prefix (stable across restarts for DHCP IP consistency) - The host DHCP server (dnsmasq) assigns an IP to the VM -- When QEMU exits, the TAP device is automatically destroyed -- VMM does not need root or `CAP_NET_ADMIN` +- On the two bridge-helper paths the TAP disappears when QEMU exits; a `netd`-created TAP is persistent and is deleted when the VMM tears the VM's networking down +- The VMM process itself needs neither root nor `CAP_NET_ADMIN` on any path; the `netd` path moves that privilege into a separate root service instead ### MAC address prefix @@ -194,13 +201,15 @@ The remaining bytes are derived from the VM ID hash. The prefix applies to all n ### Mixing networking modes -Bridge and passt VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: +Bridge and user-mode VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: ```bash -# Global default is bridge, but deploy this VM with passt -vmm-cli.py deploy --name my-vm --image dstack-0.5.6 --compose app.yaml --net passt +# Global default is bridge, but deploy this VM with user networking +vmm-cli.py deploy --name my-vm --image dstack-0.5.6 --compose app.yaml --net user ``` -### vhost-net and TDX +### vhost-net and multiqueue + +Bridge NICs can run on the host kernel's vhost-net data plane and expose several virtio-net queue pairs. Both are off by default and enabled per node or per VM — see [network-data-plane.md](network-data-plane.md) for the knobs, the enablement checklist, the mode support matrix, and how to pick a queue count. -vhost-net (kernel data plane offload for virtio-net) is **not enabled** for bridge mode. TDX encrypts guest memory, which prevents the host kernel from performing DMA-based packet offload. The default QEMU userspace virtio backend is used instead. +vhost-net works in a TDX guest: the virtio rings and buffers live in shared, unencrypted memory so that a host-side backend can reach them, which is the same mechanism `vhost-vsock-pci` has always relied on. diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index 942146f0f..6cc2724cf 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -14,8 +14,11 @@ host mechanism. The measurable acceptance criteria are: -- `network_filter = "none"` preserves the existing QEMU `-netdev bridge` - behavior and does not require `netd` or libvirt. +- `network_filter = "none"` installs no nwfilter binding. It still uses `netd` + for any NIC with more than one queue pair, and a `tap` netdev behind + `qemu-bridge-helper` whenever vhost is on; only a single-queue, non-vhost + bridge NIC keeps the historical `-netdev bridge` path with no `netd` or + libvirt dependency. - `network_filter = "libvirt"` creates the TAP and filter binding before QEMU is submitted to Supervisor, and uses QEMU `-netdev tap`. - A failed TAP or filter setup prevents QEMU from starting and rolls back all @@ -53,8 +56,11 @@ allowed_macvtap_parents = [] Macvtap is excluded from `allowed_network_modes` by default. Empty bridge and macvtap-parent allowlists prevent RPC callers from overriding the respective -node defaults. If macvtap is explicitly enabled, callers may select only a -parent in `allowed_macvtap_parents`; the macvtap forwarding mode always comes +node defaults. If macvtap is explicitly enabled, callers may select a +parent listed in `allowed_macvtap_parents`, the node's own configured parent, or +one this VM already holds — restating a value the node would have supplied +anyway grants nothing new. The same applies to `bridge_name` and +`allowed_bridges`. The macvtap forwarding mode always comes from `[cvm.networking].macvtap_mode` and cannot be selected through deployment RPCs. These allowlists authorize attachment targets; an nwfilter is not a substitute for that authorization. @@ -82,7 +88,8 @@ For libvirt mode, startup is: 2. Create the TAP for the configured QEMU UID and attach it to the bridge. 3. Create a libvirt nwfilter binding for the TAP. 4. Bring the TAP up and return success. -5. Start QEMU directly with `-netdev tap,script=no,downscript=no`. +5. Start QEMU directly with `-netdev tap,script=no,downscript=no`, carrying + `vhost=on|off` and, above one queue pair, `queues=N`. Teardown stops QEMU first, removes the binding, and deletes the TAP. Operations are serialized by `netd`. The design intentionally does not add ownership @@ -100,9 +107,9 @@ validated by libvirt. ## Deployment modes -Production should run one shared service. `netd` reads only the `[netd]` -section, so its root-owned configuration can be small and independent of every -VMM instance: +Production should run one shared service. `netd` reads the `[netd]` section, +plus `cvm.network_filter.mode` if the file has one, so its root-owned +configuration can be small and independent of every VMM instance: ```toml # /etc/dstack/netd.toml @@ -110,8 +117,33 @@ VMM instance: socket = "/run/dstack/netd.sock" socket_mode = 0o660 libvirt_uri = "qemu:///system" + +# Required here because this file has no [cvm] section for netd to read the +# node's policy from. +[netd.network_filter] +mode = "libvirt" +filter = "clean-traffic" +parameters = {} ``` +`[netd.network_filter]` is netd's own copy of the invariant, not a convenience. +netd is the privileged side of the socket, and anything that can reach the +socket can ask for an unfiltered TAP on a host bridge — a request a filtering +node has to refuse in the daemon rather than in its caller. When netd and the +VMM share one `vmm.toml`, leaving it unset derives it from +`[cvm.network_filter]` so the two cannot drift apart; a malformed section is a +startup error rather than a silent fallback to "filter nothing". + +The request says only *whether* to bind a filter, never which one. A caller that +named the filter could name `allow-arp`, which contains no drop rule at all, or +pin `clean-traffic` to the gateway's MAC and IP through its parameters, and +still satisfy a policy that asked for "some filter". + +A macvtap parent is refused when filtering is required and the parent is a host +bridge or is enslaved to one: nwfilter does not apply to macvtap, so that +request is the same unfiltered access to the same segment, spelled with a +different operation. + Production deployments can use systemd socket activation. The socket unit owns the filesystem mode and ownership; `netd.socket_mode` applies only to the standalone bind path. @@ -162,10 +194,37 @@ sudo dstack-vmm --config ./vmm.toml \ --netd-socket /run/dstack-dev/netd.sock ``` -User networking and bridge networking with `mode = "none"` never connect to -`netd`. Libvirt mode fails closed if `netd` is unavailable. - -Filtered TAP netdevs currently set `vhost=off`. This keeps the initial backend -on the directly bound TAP path and avoids adding `/dev/vhost-net` permissions -to the QEMU user. It is a deliberate security-first throughput tradeoff; a -future configurable vhost mode requires equivalent filter integration tests. +User networking never asks `netd` to build an interface; the VMM still opens a +short liveness-probe connection to the netd socket on every launch and when +describing a stopped VM. Libvirt mode fails closed if `netd` +is unavailable. Bridge networking with `mode = "none"` connects only when it +needs more than one queue pair, as described below. + +Filtered TAP netdevs follow the node's `vhost` and `queues` settings like any +other TAP-backed NIC (see [network-data-plane.md](network-data-plane.md)). The +nwfilter binding is installed on the host TAP interface, so packets traverse it +whether they were written by QEMU or by a vhost worker; filtering is unaffected +by the data plane choice. Enabling vhost does require the QEMU user to be able +to open `/dev/vhost-net`. + +`netd` also creates the TAP for unfiltered bridge NICs that ask for more than +one queue pair, because `qemu-bridge-helper` returns a single descriptor and +cannot create a `multi_queue` device. Those TAPs carry no nwfilter binding, so +a multiqueue bridge node needs `netd` even when `network_filter.mode = "none"`. + +An empty filter name is what selects that unfiltered TAP, so `mode = "libvirt"` +with an empty `filter` is rejected at config load rather than quietly producing +an unbound TAP. + +Removal carries the same distinction: the VMM tells `netd` whether the interface +it is asking about was created with a binding, from a record made when it was +built rather than from configuration that may have changed since. A binding it +was told about must be gone before `netd` returns; otherwise `netd` still asks +libvirt to clear one — an interface name is reused by the same VM, and a +leftover binding's rules would be inherited — but a `libvirtd` it cannot reach +is a warning rather than a failure. So a node with `virsh` installed and no +running `libvirtd` can create and destroy multiqueue TAPs. The flag defaults to +true on the wire, so an older VMM's removals still drop their bindings. + +`netd` requires the `virsh` binary to be present whatever the filter mode; it is +`libvirtd` that unfiltered work does not need. diff --git a/docs/macvtap-networking.md b/docs/macvtap-networking.md index 544408d31..362de4e87 100644 --- a/docs/macvtap-networking.md +++ b/docs/macvtap-networking.md @@ -19,6 +19,9 @@ Configure a NIC through node configuration or an authorized VMM RPC request: `parent` must name an existing host interface. `macvtap_mode` may be `private`, `bridge`, `vepa`, or `passthru`; an empty value selects `private`. +Macvtap NICs also honour the `vhost` and `queues` settings described in +[network-data-plane.md](network-data-plane.md); netd creates the interface with +matching hardware queues and the launcher opens `/dev/tapN` once per queue. The configured netd socket permissions apply in the same way as for libvirt-filtered bridge networking. @@ -49,8 +52,9 @@ and the same deterministic MAC address passed to QEMU. Netd then: 4. reads its kernel-assigned ifindex and waits for `/dev/tap`; and 5. returns that runtime device path to the VMM. -The per-VM launcher opens the character device, places it at the fd referenced -by QEMU's `-netdev tap,fd=...` argument, and then execs QEMU. This keeps device +The per-VM launcher opens the character device once per queue pair, places the +descriptors at the fds referenced by QEMU's `-netdev tap,fd=...` (or `fds=...`) +argument, and then execs QEMU. This keeps device paths out of persistent VM configuration, works with both Supervisor and systemd process managers, and does not pass network fds through `sudo`. diff --git a/docs/network-data-plane.md b/docs/network-data-plane.md new file mode 100644 index 000000000..57f0134a1 --- /dev/null +++ b/docs/network-data-plane.md @@ -0,0 +1,306 @@ +# virtio-net data plane tuning + +Every CVM NIC has two knobs that decide how many packets it can move: whether +the host kernel's vhost-net data plane is used, and how many virtio-net queue +pairs the device exposes. vhost is set per node and overridable per VM; queue +pairs have no node-wide setting at all, for the reason given under +Configuration. + +## Why it matters + +Without vhost-net, QEMU drains every received packet on its single main-loop +thread. That thread is the ceiling, and it does not grow with vCPUs: + +``` +maximum packets per second ≈ 1 core ÷ per-packet main-loop cost +``` + +The per-packet cost varies with traffic shape — a few microseconds for uniform +synthetic streams, tens of microseconds for bidirectional short-connection +traffic — so the ceiling is a property of the workload, not a fixed number. +What is fixed is the shape of the failure: throughput climbs normally until the +main loop saturates at 100% of one core, then packets are dropped at the TAP +before they ever reach the guest. Guest-side counters stay clean, which makes +the cliff easy to misdiagnose as a network problem. + +Guest-side outbound traffic uses the same thread, so a busy guest pays the +cost twice over. + +`vhost=on` moves that work into the host kernel. That returns a whole core, but +it relocates the ceiling rather than removing +it: packets now arrive faster than a single guest receive queue can drain, and +the drops reappear at a higher rate. More queue pairs is what removes them, +which is why enabling vhost also enables a queue count that follows the VM's +vCPU count — the two travel together. + +vhost is **off by default** and enabled per node (or per VM). Two things make +it an opt-in rather than a default: turning it on changes the virtio-net device +of every bridge/macvtap VM on its next boot, and it requires `/dev/vhost-net` +to be accessible to the account QEMU runs under, which the VMM cannot verify on +the operator's behalf — see [Enabling vhost on a node](#enabling-vhost-on-a-node). + +## Configuration + +```toml +[cvm] +# Ceiling for both the default and what a deployment may request. +max_net_queues = 16 + +[cvm.networking] +mode = "bridge" +bridge = "dstack-br0" +vhost = true +``` + +Queue pairs are not a node setting. With vhost on they default to the VM's vCPU +count, capped at 16, because the useful number follows the VM rather than the +host — the guest driver uses at most one queue pair per vCPU. A deployment +overrides that per VM, up to `max_net_queues`. + +Raising `max_net_queues` above 16 widens what a deployment may ask for without +moving the default's cap, so a larger VM never silently acquires a worse +default. Lowering it below 16 does lower the default too, because a node that +refuses a request for four queue pairs should not hand out sixteen by itself. +The hard ceiling from any source is 64. + +Without vhost the default is a single queue pair. The QEMU main loop drains +every queue on one thread, so extra queues buy little while still costing a +netd interface, more MSI-X vectors, and a changed guest device. An explicit +queue count is still honoured without vhost, since that combination is a +deliberate request rather than a default. The two defaults travelling together +also means a node that never sets `vhost` keeps building the device its VMs +have always had. + +A VM overrides either value at deploy time, and `UpdateVm` changes them +afterwards — the new values apply from the VM's next boot: + +```bash +vmm-cli.py deploy --name my-vm --image dstack-0.5.9 --compose app.yaml \ + --net bridge --net-queues 4 +vmm-cli.py deploy --name latency-vm --image dstack-0.5.9 --compose app.yaml \ + --net bridge --net-no-vhost + +# retune an existing VM +vmm-cli.py update --net-queues 8 --net-vhost + +# stop pinning either value and follow the node and the vCPU count again +vmm-cli.py update --net-queues auto +vmm-cli.py update --net-vhost-default + +# stop pinning the backend too, and follow whatever the node runs +vmm-cli.py update --net default +``` + +Every pin has an un-pin. A VM keeps reporting whatever it pinned, and the +deployment RPC accepts a VM's own held values back even after the node's +policy moves, so a read-modify-write update never strands a VM. + +The web UI exposes both per NIC in the deploy and update dialogs, alongside the +networking mode. Both fields are also on `NetworkingConfig` in the deployment +and update RPCs. A request that +sets only `vhost`/`queues` keeps the node's own networking mode, so tuning does +not force a caller to restate — or be allowed to choose — a backend. `queues` is +rejected above the node's `max_net_queues`; `vhost` is not otherwise restricted, +since it only affects the requesting VM. `GetMeta` reports +`networking.max_queues` so a client can present the real bound. + +The data plane settings are recorded only when a deployment asks for them. +Leave one out and it stays owned by the node, so changing `[cvm.networking]` +later — including setting `vhost = false` to roll the whole node back — still +reaches VMs deployed with some other networking override. + +Naming a backend is different: it pins that NIC's *backend*, resolved at +deployment. Its mode, and the bridge or macvtap parent that names it, are fixed +for the life of the VM, so a later edit to those fields in `[cvm.networking]` +does not move it to another segment. Nothing else is pinned — the MAC prefix, +the user-mode subnet and DHCP start, and the macvtap forwarding mode stay node +settings and are re-read at every launch, so editing them changes every VM's +next boot, including its MAC and therefore its DHCP lease. A request that only +tunes pins nothing at all, including the backend it inherited. + +`GetInfo` reports that configuration back, and both `vmm-cli.py update` and the +web UI read it, change one field, and resend the rest. Two things follow. A +request may name a bridge or macvtap parent the node itself configured even when +the allowlists are empty: leaving the field out already yields exactly that +value, so echoing it grants nothing policy was withholding. And an update may +restate whatever its own VM already pinned, so that moving the node's default +out from under a VM does not leave that VM's configuration unsendable. A NIC +that inherited its backend reports an empty mode, which is the same thing it was +deployed with. + +Neither field reaches the CVM's measurement. The measured VM configuration the +VMM controls covers the OS image, the vCPU and memory counts, several QEMU +layout flags, the *number* of NICs, and `mr_config_id`; the queue count and the +vhost state are not part of it, so retuning a NIC does not change app identity +or require an on-chain update. Adding or removing a NIC does: the NIC count +changes the guest's ACPI tables and therefore RTMR0. + +## Enabling vhost on a node + +Setting `vhost = true` in `[cvm.networking]` is a node-wide behaviour change: +every bridge or macvtap VM that has not pinned its own data plane gets a +different virtio-net device on its next boot — `vhost=on`, `mq=on` with +vCPU-scaled queue pairs, and the matching MSI-X vector count. The device is not +measured, so attestation and app identity are unaffected. Before flipping it: + +1. **Verify `/dev/vhost-net` is accessible to the account QEMU runs under.** + It is `root:kvm 0660` on Debian-family hosts, where adding the account to + the `kvm` group suffices, and `root:root 0600` on several others. If the + account lacks access, QEMU exits at launch and every affected VM stops + restarting. The VMM warns at startup when its own access fails, but it + cannot refuse on that basis — QEMU need not share its credentials. + +2. **Restart `netd` before or together with the VMM.** Multiqueue bridge NICs + are prepared by `netd`, and the VMM checks that `netd` echoes the queue + count it built. An older `netd` fails that check; the launch is rolled back + and fails with the reason in the VMM log, but the VM does not start until + `netd` is upgraded. + +3. **Roll back by setting `vhost = false`.** The node value reaches every VM + that did not pin `vhost` explicitly, from its next boot; a VM that pinned + `vhost = true` keeps it until updated. + +## What each mode supports + +| Mode | netdev | vhost | queues > 1 | +|---|---|---|---| +| `user` | `user,...` | no backend | not supported | +| `bridge` | `tap,ifname=` via netd, else `tap,br=,helper=`, else `bridge,br=` | yes | yes, through netd | +| `bridge` with libvirt filtering | `tap,ifname=` | yes | yes, through netd | +| `macvtap` | `tap,fd=` / `tap,fds=` | yes | yes | +| `custom` | operator's own string | operator's own string | no, not settable | + +QEMU's `bridge` netdev accepts neither `vhost=` nor `queues=`, so enabling +vhost switches bridge mode to a `tap` netdev driven by the same setuid +`qemu-bridge-helper`. The VMM still needs no network privileges. The helper has +no compiled-in default path for the `tap` netdev, so the VMM probes the known +distribution locations; set `cvm.qemu_bridge_helper` if yours is elsewhere. If +no helper is found the NIC falls back to the non-vhost `bridge` netdev with a +warning, because a node-wide setting must not stop a node from booting VMs +that never asked for it. + +The helper returns exactly one descriptor, which is why more than one queue +pair in bridge mode is created by `netd` instead: it adds a persistent +`multi_queue` TAP that QEMU then opens once per queue. `netd` requires the +`virsh` binary to be installed even when nothing is filtered, though it does +not require a reachable `libvirtd`. That applies whether or +not libvirt filtering is on, so a bridge node needs `netd` to get the default +queue count (see [libvirt-network-filter.md](libvirt-network-filter.md)). +Without it, bridge NICs fall back to a single queue pair with a warning rather +than failing to launch; a VM that asked for a queue count explicitly still +fails, so the caller learns their request was not met. `netd` is probed by +connecting, not by looking for its socket file, because a `netd` that died +leaves the socket behind. One-shot `dstack-vmm run` has no netd lifecycle at +all and behaves like a node without it. `netd` reports back the +queue count it created, and the VMM refuses to launch on a mismatch — a `netd` +deployed separately as a root service can be older than the VMM asking it for +multiqueue, and QEMU would otherwise reject the interface from inside the +per-VM launcher. + +For macvtap, the per-VM launcher opens the `/dev/tapN` character device once +per queue pair and hands QEMU the descriptors as `fds=`. `netd` creates the +interface with matching `numtxqueues`/`numrxqueues`. + +Custom mode owns its whole netdev string, including any `vhost=`/`queues=` +options, and its guest device stays single-queue: the VMM cannot edit that +string, so it has no way to make a multiqueue device line agree with it. For the +same reason `GetInfo` reports no vhost state and no queue count for a custom +NIC, rather than asserting the resolved defaults over a string it never read. A +hand-written multiqueue netdev will not pair with a multiqueue guest device +today. + +Naming a backend that cannot carry vhost or a queue count, and then asking for +one, is refused — the request is yours to correct. Inheriting such a backend is +not, because the node chose it and may choose another tomorrow; the request +reads as off, or as one queue pair, until then. + +## Choosing a queue count + +The default suits bandwidth-bound workloads. Latency-sensitive ones should ask +for fewer: more queues spread receive processing over more vCPUs, and under TDX +a cross-vCPU wakeup costs an IPI and a VM exit. Measured on one 8-vCPU TDX CVM, +changing only the guest's channel count: + +| Queue pairs | Short-connection throughput | +|---|---| +| 1 | 22.3k conn/s | +| 2 | ~20k conn/s | +| 4 | 15–21k conn/s | +| 8 | 6.2–7.7k conn/s | + +The same CVM with 8 queues moved 3.0 Mpps of 64-byte UDP with no loss, against +roughly 600k with one queue. The trade is real in both directions, so a VM +serving many short connections should set `--net-queues 1` and measure. + +A VM with fewer vCPUs than queues leaves the extra pairs idle — `ethtool -l +eth0` reports the smaller number. An explicit over-provision is not rejected at +deployment, because `vmm-cli.py resize` can raise the vCPU count later. + +Queue pairs also cost guest memory — each RX ring keeps 256 page-sized buffers +posted, about 1 MB per queue pair plus per-queue NAPI and socket state — but at +any realistic RAM/vCPU shape this is noise. Pushed to a shape no deployment +uses (1 GB of RAM with 16 vCPUs, so 16 queue pairs by default), sustained load +did produce atomic order-0 page-allocation failures in RX refill +(`try_fill_recv` in the guest log); 2 GB at the same shape ran clean. Since the +queue default follows the vCPU count and a VM with that many vCPUs carries far +more memory in practice, this needs no tuning — it is recorded here so the +symptom is searchable. The TDX bounce-buffer pool is not a constraint either: +the guest kernel sizes swiotlb at 6% of RAM clamped to [64 MB, 1 GB] with no +`swiotlb=` parameter, while peak demand is bounded by ring size at about 2 MB +per queue pair — a deliberately undersized 32 MB pool sustained full +multiqueue line rate with zero `swiotlb buffer is full` events. + +`vectors` is derived, never configured: `2N + 2`, one vector per queue +direction plus config and control. One queue pair emits no `mq=on` or +`vectors=` at all, leaving the guest device line byte for byte identical to the +one before this feature. The `-netdev` half does change wherever vhost is on, +since that is what selects the backend. + +## Requirements + +The account running QEMU must be able to open `/dev/vhost-net`, which is +`root:kvm 0660` on a stock host — add that account to the `kvm` group. The +`vhost_net` module autoloads on first open. + +`GetInfo` reports the data plane each interface actually got, so a bridge NIC +that fell back for want of a helper reads as `vhost: false` rather than +advertising something it is not using. For a VM that is not running there is no +interface to describe, so it reports what the next launch would build instead -- +the same calculation, against the node configuration and manifest as they stand +now, rather than the ones a finished boot ran under. + +If that account lacks access, QEMU exits at startup and the VM never boots — +there is no fallback to the userspace backend at this point, on any QEMU +version (verified on 8.2.2 and 10.2). What the per-VM launcher log shows +depends on the version: QEMU 8.2 prints `warning: tap: open vhost char device +failed: Permission denied` (once per queue) and then dies on `net/net.c:1185: +net_client_init1: Assertion 'nc' failed` — an upstream bug +([qemu#1486](https://gitlab.com/qemu-project/qemu/-/issues/1486)); later +versions exit cleanly with `Could not open '/dev/vhost-net'`. Grep for either. +The VMM does not refuse a launch over this: QEMU need not share the VMM's +credentials, so a refusal based on the VMM's own access would block deployments +the host can run. It warns instead — when the device node is missing outright, +and when the VMM's own open is denied, since QEMU usually does share its +account. + +QEMU does have a *runtime* fallback, at a different failure point: once the +netdev initialized with vhost, a later `vhost_net_start()` failure at guest +driver activation logs `unable to start vhost net: : falling back on +userspace virtio` and keeps the NIC working on the userspace data path. That +path is reachable only after `/dev/vhost-net` was opened successfully at +launch, so an access problem never lands there. If it does fire, it is the one +case where `GetInfo` can overstate the data plane — the interface reports the +vhost state the launch settled while the packets take the userspace path — and +that QEMU log line is the indicator. + +vhost-net works normally in a TDX guest: the virtio rings and buffers live in +shared, unencrypted memory precisely so a host-side backend can reach them. +This is the same mechanism behind `vhost-vsock-pci`, which dstack has always +used. + +On host kernels older than 6.4 the vhost worker is a free-standing kernel +thread: it is attached to the owner's cgroups, so `cpu.max` and cgroup +accounting do apply, but it is outside QEMU's thread group and so invisible to +`top -H` and to anything reading `/proc//task`. Since 6.4 it is a +`vhost_task` inside that thread group and shows up everywhere the VM's other +threads do. From 9cce98e0ae248859b451bafeece0966a94ab5931 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 5 Sep 2026 10:55:48 +0800 Subject: [PATCH 3/5] feat(vmm): give port mappings a NIC, and carry them to netd (#1154) * feat(netd): carry host port mappings and the VM workdir to netd `port_map` is implemented as QEMU `hostfwd=` entries on a user-mode netdev. A bridge NIC has none, so `configure_networking` finds no interface to hang them on and emits none -- with no warning, no validation, and `GetInfo` still reporting the ports as though they worked. A VM moved from user mode to a bridge silently loses every published port. The VMM cannot fix that itself: it runs without CAP_NET_ADMIN by design, and the userspace forwarder that used to cover this was removed in e2e607fd4 because proxying on the host hands back the per-packet cost that leaving user mode was meant to escape. netd is privileged and is the only component that sees every VMM instance on a host, so it is also the only one that can arbitrate a host port between them. So state the requirement rather than assume it is met. `ingress` on `prepare_bridge` carries protocol, host address, host port and guest port. The host address is not decoration -- an admin port bound to loopback and a published one differ only there. Every field is caller-named, which is what `bridge`, `mac` and `queues` already get and the opposite of `filtered`. The line is whether netd can check what it is handed: an nwfilter name cannot be checked for whether it filters anything, while a host port is a closed space a node policy can be stated over. Naming is not deciding; which ports may be handed out stays netd's own configuration, exactly as `allowed_bridges` governs the bridge a caller names. The netd here builds interfaces and does not forward ports. It says so by leaving `ingress` out of its response, which is the reading `queues` already has: absent separates "this netd does not do that" from "nothing was asked for". The VMM warns on that rather than failing, because a VM deployed before this has been running with its ports dropped and refusing to launch it now would turn a silent misconfiguration into an outage on upgrade. Also add `workdir` to both prepare operations. Untrusted and never read for a decision, it is there so an operator reading netd's log can get from an opaque TAP name back to the VM that asked for it. No new operations, and nothing removed. * feat(vmm): give port mappings a NIC, and carry them to netd Three things that are one thing: a port mapping had no way to say which NIC it uses, the NIC it silently got was often the wrong one, and on a bridge it got nothing at all. `port_map` is implemented as QEMU `hostfwd=` entries, and those need a user-mode netdev: let hostfwd_index = self.prepared.networks.iter() .position(|n| n.mode == NetworkingMode::User); Multi-NIC (#756) made that a choice, and it has been made silently ever since. A bridge NIC for external traffic beside a user-mode NIC for management -- the topology multi-NIC was added for -- puts every published port on the *management* NIC: the traffic reaches the guest, but over slirp, bypassing whatever the bridge NIC's nwfilter was there to enforce and hiding the client's address behind the slirp gateway. A second user-mode NIC can never publish anything, because `position` returns the first. And with no user-mode NIC at all there is no `hostfwd_index`, so every mapping is dropped with no warning while `GetInfo` keeps reporting the ports as though they worked. So say it. `PortMapping.nic_index` names the NIC a mapping enters through, `@` on the CLI, unset resolving to the first user-mode NIC and failing that the first bridge NIC. Existing VMs keep their behaviour exactly wherever a user-mode NIC exists; where none does, ports now have somewhere to go instead of nowhere. One mapping resolves to exactly one NIC, and that NIC's backend decides the mechanism: `hostfwd=` for user mode, netd for a bridge. Both sides read the same resolution, so no host port can be claimed twice. The VMM cannot forward a bridge NIC's ports itself -- it runs without CAP_NET_ADMIN by design, and the userspace forwarder that used to cover this was removed in e2e607fd4 because proxying on the host hands back the per-packet cost that leaving user mode was meant to escape. netd is privileged and is the only component that sees every VMM instance on a host, so it is also the only one that can arbitrate a host port between them. `ingress` on `prepare_bridge` carries the requirement there. Every field of it is caller-named, which is what `bridge`, `mac` and `queues` already get and the opposite of `filtered`. The line is whether netd can check what it is handed: an nwfilter name cannot be checked for whether it filters anything, while a host port is a closed space a node policy can be stated over. Naming is not deciding. The netd here builds interfaces and does not forward ports. It says so by leaving `ingress` out of its response -- the reading `queues` already has, where absent separates "this netd does not do that" from "nothing was asked for". The VMM warns rather than failing, because a VM deployed before this has been running with its ports dropped and refusing to launch it now would turn a silent misconfiguration into an outage. Also add `remove_all`. Teardown by identity only reaches the NIC indices its caller still has a record of, and that record is written after the interface exists: a VMM killed in between leaves a TAP nothing on disk points at, a lost record reads as "nothing to remove", and a manifest that lost a NIC leaves an index the list no longer reaches. netd derives every name a VM could occupy instead, bounded by what an identity may say. The VMM sweeps before preparing a launch as well as on stop, so a launch is self-healing regardless of what the record says, and an unreachable netd no longer fails a stop. And `workdir` on both prepare operations: untrusted, never read for a decision, there so an operator reading netd's log can get from an opaque TAP name back to the VM that asked for it. * fixup! feat(vmm): give port mappings a NIC, and carry them to netd * fix(vmm): ask netd what it can do instead of guessing from failures `remove_all` is an operation, and an operation a netd does not have answers the same way one that failed does: `ok: false` with a message. The VMM read that as a failed sweep, so against any netd older than this branch a stop returned an error *and* left every interface behind -- strictly worse than the per-NIC removal it replaced. `removed` was shaped to carry the other half of the signal and then discarded with `unwrap_or_default`, which turns "this netd does not sweep" into "this VM had nothing to remove": the exact conflation `queues` and `ingress` are shaped to avoid. So ask. `hello` answers a capability set, is served before the operation lock so a probe never waits behind a collection, and is bounded well below the request timeout because every teardown asks it. A netd that predates it answers an error, which is still an answer: reached, and old. The three states a caller has to act on -- absent, old, capable -- are distinguishable here and nowhere else. Teardown is then unconditional and non-fatal, which is one decision made twice. It was gated on `needs_netd_interface`, which reads the VM's *current* backend: a VM whose NIC was a bridge when its TAP was built and is user-mode now skipped the release for interfaces that exist, and neither its stop nor its next launch would ever reach them again. And it returned an error a stop propagated, so a netd outage became a fleet that could not be stopped. Nothing is lost by not failing: a launch releases before it prepares, so the next one is self-healing. `is_unreachable` never returned true. The marker is attached with `context`, which makes it the context *of* a chain link rather than a link, so the walk over `chain()` could not see it -- every "an unreachable netd is not a failure" branch in this crate was dead. `downcast_ref` sees it. And a fake netd to talk to. The VMM's side of this protocol had no test at all, because every path looked like it needed a privileged daemon; it needs something that answers on a socket. * feat(netd): record who an interface belongs to, on the interface Deriving a name answers "where is this VM's interface". It cannot answer "whose is this interface", and that is the question a leak is made of: a VM whose directory was deleted, a VMM instance that was decommissioned, an interface built before an upgrade. A digest is not reversible, so nothing on the host could attribute one -- and with several VMM instances sharing a netd, nothing could even tell whose it was to collect. So write it down where it cannot drift: `ip link set dev alias`. The kernel holds it for exactly the interface's lifetime. A file under /run would be a second thing to keep in step with the first, and a record that got out of step -- written after the interface, lost with the directory -- is the failure being fixed, not the fix. The record is a hint, never an authority. Anything that can reach this socket can assert an identity, and the interface name is a digest of one, so a record is believed only when re-deriving the name from it reproduces the name it is written on. Forged, truncated, ambiguous and absent all fail that one check and land in the same bucket, which is the bucket a collection has to treat conservatively anyway. `list` then enumerates: the reserved `dt` name space, narrowed by the kernel's own answer about what kind of device it is, so netd will not offer up something it did not create. Orphaned nwfilter bindings are listed too -- a binding outlives the interface it was bound to, so an inventory that only looked at interfaces would miss the one piece of state that survives them. Absent rather than empty on a netd that cannot enumerate: "I hold nothing" and "I cannot say" are not the same answer. `dstack-vmm netd list` reads it. An operator could not previously see what netd held at all. One new refusal: an identity whose record would exceed the kernel's 255-byte alias, or whose instance ID contains the separator. Both would produce an interface nothing can attribute, which is the state this exists to stop creating. Real identities are a `path-` digest and a VM ID: about sixty bytes. * feat(vmm): collect host interfaces no VM claims Per-VM teardown reaches only what its caller can still name, and a leak is made of exactly the interfaces nothing names any more: a VM removed while the VMM was down, a workdir deleted by hand, a teardown that raced a netd outage and whose VM no longer exists to retry it. No amount of care at the per-VM call sites reaches those, because the call site is gone. `gc` compares what netd holds against the VMs a VMM instance has. The live set is every VM it has, running or not: a VMM restarts under VMs that keep running, and collecting by what is running would delete their networking out from under them. It runs at startup after the VMs are loaded and before the API is served -- the set is a snapshot, and a VM created between taking it and acting on it would be in netd's listing and not in the set -- and then on an interval, for what accumulates while the VMM is up. Three rules, and the second is why interfaces carry a record at all: - recorded as another instance's: never touched. Two VMM instances share one netd, and without the record a collection cannot tell that instance's running VM from garbage. - recorded as ours, for a VM we no longer have: collected. - no record that checks out: kept by default, and reported. It is not nobody's -- it is an interface from before ownership was recorded, or from another netd. `collect_unattributed` opts in where the operator knows nothing else creates interfaces in netd's name space. The upgrade is safe by construction: a collection derives the names its own live VMs would occupy and keeps those even when told to collect unattributed, so a fleet running from before this existed survives the first pass, and each interface gains a record the next time its VM launches. Both passes are bounded. Under the operation lock and inside a serialized accept loop, an unbounded pass is not slow, it is an outage -- one hung `virsh` per interface holds every other VM's prepare behind it while the caller that asked has long since timed out. A deadline stops the pass and reports what it did; libvirt is asked once per pass rather than once per interface, because asking again after it has failed is how a bounded pass becomes an unbounded one. `dstack-vmm netd remove-vm` for a VM whose VMM will never ask again. * feat(vmm): report which ports a VM actually publishes A port mapping is a request. Which NIC carries it decides who answers it: QEMU's `hostfwd` on a user-mode NIC, always; the node's netd on a bridge NIC, which -- like the netd in this repository -- may not forward host ports at all. `GetInfo` reported the request either way, so a VM could list published ports that nothing on the host forwarded any traffic to. So keep the answer. A prepare's `ingress` response is recorded on the NIC beside `netd_interface` and `device`, and `PortMapping.published` reports it: absent for a VM that is not running or a VMM that predates the field, true only where something actually publishes the port. Held to it per mapping, not per interface -- a netd may refuse one port out of a set, and the mapping that lost is the one worth naming. And refuse at deployment what a launch can only warn about. Two ways to have nowhere to go: a mapping that resolves to no NIC at all, and one that resolves to a bridge NIC on a node whose netd does not forward. Refusing costs nothing there, because nothing is running on the answer yet; refusing at launch would turn a VM that has been running with its ports dropped into an outage on upgrade. An update refuses only when it moved the ports or the networking, so a VM deployed before the node could answer for its ports stays editable in every other respect. The ownership contract that makes this collectable is now written down: a host port a netd publishes is owned by the interface and released with it, and there is deliberately no operation that releases one separately. A reservation outliving its interface could never be attributed to a VM again, because the interface is the only thing that carries a record. * test(netd): put the ownership record next to the kernel, and document it Everything else here reasons about strings. This creates a real TAP, reads the alias back off it, finds it by enumeration, and checks that the guards refuse what they are meant to: a device with one of netd's names that netd did not create, and a record that does not re-derive the name it is written on. It refuses to run in the host's network namespace. Unsharing one from inside the test is not enough -- `/sys/class/net` keeps showing the old namespace until sysfs is remounted, which is most of what `ip netns exec` does -- so it asks to be put in one instead, and says so in the doc comment. That also means it can never touch a real node's interfaces, including when it fails. The docs gain what an operator needs to act on any of this: how to read an interface's owner, how a collection decides, why an interface with no ownership record is kept rather than deleted, and why upgrading needs no migration step. * fix(vmm): claim a VM's interfaces from its directory, not just from what loaded A VM whose manifest is corrupt or whose image is missing fails to load and is only logged -- but its QEMU may well still be running. Collecting by what loaded would delete a running VM's networking over a file the VMM could not parse. A directory is enough of a claim, and what has been removed for real leaves none behind. An unreadable VM directory is not an empty one either: read as empty it would offer every interface on the host up for collection, so it is an answer the reconciliation declines to act on at all. * fix(vmm): refuse an instance ID no interface could be recorded as belonging to At startup rather than at the first launch. The VMM derives one that is always valid; an operator who configured their own learns here rather than from the first VM that fails to get a NIC. * docs(vmm): say what cvm.instance_id now decides It is what netd records on every host interface this VMM asks for, and what a collection uses to tell this instance's interfaces from another's. Two VMMs sharing one value on one host would each collect the other's running VMs. * fix(cli): carry the new port mapping field through the client crates Additive on the wire and in the API: a client states a request, and `published` is the server's answer to it. * fix(netd): stop a busy netd from reading as an absent one The accept loop served one connection at a time and `handle_request` blocks, so answering `hello` "before the lock" bought nothing: the connection was not *accepted* until whatever netd was doing finished. A collection may run for twenty seconds and a single `virsh` for thirty, while the capability probe gives up after five -- so any netd doing real work looked absent, and an absent netd is one whose teardown the VMM skips and whose deployments it refuses with "this netd does not forward host ports". Both wrong, and both introduced by the probe this branch added. One task per connection, with the blocking half on `spawn_blocking`. Serialization still holds, and now holds where it is actually stated: the operation lock is an flock, which contends between two open descriptions in one process exactly as it does between processes. What the single-connection loop added on top was head-of-line blocking and nothing else. The client side stops asking a question it does not need to ask. A release asks netd to release, rather than asking whether it may: the operation cannot be misread as absence, and only a refusal -- netd answering -- is worth a round trip, which is also exactly when the fallback matters. That removes the probe from every stop, and removes the case where the netd released one commit before this one, which does implement `remove_all`, was told it could not and given eight per-NIC removals instead. Reconciliation reads the same way. The one probe left in front of an action is the one that decides whether to act on the *absence* of an ownership record, which means nothing unless netd writes one -- so `attribution` is finally read where its documentation already claimed it was. Also: an orphaned nwfilter binding is now collected by default. It is the one unattributable thing that is unambiguously dead -- a record can only live on an interface, so it can never gain one, and nothing is using a binding with nothing to bind to. Left to the conservative default it was the single leak in this design that nothing could ever collect, since the VM it belonged to is gone and not even an operator could name it. And three narrower ones: a directory entry that cannot be read no longer silently drops a VM from the set that claims interfaces (the same care the `read_dir` error already got); a binding that answers on a different address than it was asked for is no longer reported as publishing the port, since an admin port on loopback and a published one differ only there; and two live VMMs configured with one `cvm.instance_id` now refuse to start rather than each collecting the other's running VMs. * fix(netd): stop the connection timeout from throwing away work it cannot cancel It wrapped the whole exchange, which reads as a bound on the request and is not one: `handle_request` is synchronous and shells out, so the timeout could not cancel it. All it did was drop the connection at thirty-five seconds while the work went on running to completion -- and then log "netd connection timed out" for a request netd in fact finished. A caller has its own deadline and has gone by then; what this cost was netd's own account of what it did. Now the timeouts bound the socket reads and writes, which is what the comment said they were for. Observed against a netd wedged in a twenty-five second helper: `hello` answers in 0.00s, a queued `list` answers at 49s rather than being cut off at 35s, and the log says what happened. * docs(vmm): note that an orphaned nwfilter binding is always collected * fix(netd): make the binding listing work, and decide collections where the lock is Two findings from review, and the second reshapes the design. `virsh nwfilter-binding-list` accepts no options at all -- `--name` is not one of them, and asking for it fails the whole call. So `existing_bindings` has always returned `None`, which was survivable until this branch made a sweep read that as "libvirt is down, skip the bindings": every stop, every pre-launch release and every removal then deleted the TAP and left the binding, permanently for a VM that was removed. Verified against virsh 10.0.0 on a node holding seven real bindings. It parses the table now, narrowed to netd's own name space so a header, a rule line or a moved column cannot produce a name; and a listing that could not be produced no longer decides whether deletions are attempted. The collection moves out of netd. A collection decided inside netd is decided against a set of live VMs that was true when the caller *sent* it -- netd runs the request when it wins the operation lock, which may be much later -- so a VM created in between is absent from the set and present on the host, and its TAP is deleted while QEMU is starting on it. That was safe at startup, where the API is not yet served, and the interval pass ran the same code with the API served. netd cannot close it: the lock that closes it is the VMM's per-VM launch lock, and netd has no way to take one. So the VMM asks `list` and decides for itself, per VM, under exactly that lock, re-reading whether it claims the VM while holding it. A launch and a collection of the same VM can no longer both believe they are alone. It also deletes `Gc`, `gc_plan`, `live_interface_names`, `collect_garbage`, `UnattributedPolicy`, `Collection`, dry runs, and the `collect_unattributed` knob: the collection is `list` plus the whole-VM sweep a stop already uses. What is left over is what nothing can attribute, and nothing decides about it -- correct, and previously a dead end, since `netd list` shows no VM to name in `remove-vm`. `netd remove-interface ` is the operator's way to say what a machine cannot work out. Also: a VM directory that is not there is no longer read as "no VMs". "Never ran" and "the volume is not mounted yet" are the same error, and only one of them means there is nothing to collect. * fix(vmm): do not cache an answer that may have been a blip `Unreachable` and `Legacy` are both produced by transient failures, and holding either for half a minute turns one blip into a deployment refused because 'this netd does not forward host ports' -- which may not be true. Only a real capability answer is worth reusing; the other two are re-asked on paths that were already making a round trip. * chore(netd): drop a test helper the collection reshape left unused * fix(vmm): decide whether a VM is running under the lock that keeps it still An audit of the stop-time removal found the removal itself sound and the *decisions* around it not. Two callers read "is it running" outside the per-VM launch lock and acted on the answer inside it, which is precisely the race the lock exists to close. `update_vm` with a networking change: the read says not-running, a launch then takes the lock, prepares TAPs and deploys QEMU, and the update takes the lock afterwards and releases the interfaces of a VM that is now running. QEMU keeps running with a dead NIC, the supervisor still reports it healthy, and nothing logs a thing. `finish_remove_vm`: it stopped the process and waited for exit entirely outside the lock. A launch that passed the `removing` check before the marker was set is already inside the lock and has not deployed yet, so the wait sees nothing running and returns at once; the launch then starts QEMU, and removal takes the lock and deletes its interfaces, its workdir and its CID. The VM runs on, invisible, until the next orphan sweep at startup. Both now take the lock first and ask afterwards. A third, smaller: an automatic restart reads the started flag off disk and only then queues a launch, which waits for the lock a concurrent stop is holding. Without a re-read under that lock, the launch resurrects a VM the operator was told was stopped. An explicit start sets the flag itself and has nothing to re-read, so only the automatic path re-checks. Two things the same audit found in the removal proper. A sweep truncated by netd's deadline reported `incomplete`, which the client dropped -- so a host with interfaces left over logged "released 3 interfaces" and nothing else; it now says so. And `Request::Remove` no longer fails when libvirt cannot delete a binding: that strictness was written for prepare, where a leftover binding blocks the creation about to happen, and at removal it only leaves the interface itself up on the bridge instead of a binding libvirt hands back on its next listing. It also now agrees with the whole-VM sweep, which was always best effort. The comment on `stop_vm` claimed reconciliation would collect what a failed release left behind. It will not: a stopped VM is still one this instance claims, so a VM that is never started or removed again keeps its interfaces. Say that instead. * fix(vmm): stop the removal lock from starving everything that waits on it Holding the per-VM launch lock across removal closed a race and opened two queues behind a wait its own comment measures in hours. A start of a VM being removed asked `removing` only after taking the lock, so it now waited out the whole removal to be told no. It asks before as well -- not instead: the marker can be set while it waits, so the answer under the lock is still the authoritative one. Reconciliation was worse. It takes the lock per VM, in sequence, so one VM stuck in removal stalled the collection of every other VM on the host -- and because the hourly task awaits it, the interval never fires again. The case is real rather than theoretical: an orphaned supervisor process cleaned up by `reload_vms` has no VM in memory and may have no directory, so it is exactly the kind of VM a collection considers dead while its removal is still polling for exit. It takes the lock without waiting now, and skips what it cannot get. That is not a compromise: a held lock means a launch, a stop or a removal of that VM is in flight, and every one of those manages the VM's interfaces itself, so waiting would be waiting for the thing that makes the work unnecessary. * fix(vmm): refuse a stop or an update of a VM being removed, before waiting on it Both took the launch lock that removal holds until the VM has exited, and neither asked whether the VM was being removed at all -- so a stop or an update issued during a removal hung for as long as the removal took, to do work the removal was already doing. Neither has an internal caller; both are RPCs, and an RPC that hangs for hours is worse than one that says why. * fix(vmm): put the CLI help back on the subcommand it describes `netd remove-interface` was inserted between `remove-vm`'s doc comment and its variant, so clap printed "Delete every interface netd holds for one VM" as the help for removing a single interface, and nothing at all for removing a VM's. Three claims that stopped being true, while their code changed underneath them: `Request::Remove`'s `filtered` field is still sent and still required on decode, but the handler no longer derives strictness from it; removal's wait is a SIGKILL teardown, not the "2+ hours" a graceful stop used to take; and what a truncated sweep leaves behind is collected by reconciliation when the caller was the removal, since there is no next launch to do it. * fix(vmm): keep the removal mark where the removal can be seen `refuse_if_removing` read a flag on the VM's entry, which the orphan cleanup never sets: `spawn_finish_remove` marks nothing when the ID has no entry, and that is exactly the case it exists for. So `finish_remove_vm` held the launch lock across a whole teardown while every operation on that ID was told it was free to proceed -- and `StartVm` then waited the teardown out with no error and no log, only to fail at the end on a VM no longer in memory, after it had already written started to disk. The mark now lives in the state, not in the entry, so an ID with no entry can carry one. It is cleared by a guard rather than by the last step of the happy path: `finish_remove_vm` has two `?`s after the wait, and a mark left behind by either is not a stale flag but a VM no operation can reach again, including the removal that would retry. `claimable_vm_ids` hands back the set it built instead of collecting it into a vector for the caller to linear-scan once per interface. * fix(vmm): hold an update to one VM against the removal of that VM `update_vm` asked whether the VM was being removed only inside the branch that changes networking, and dropped the launch lock at the end of that branch. What follows is the part that writes: `put_manifest` creates the directory it writes into. An update that resumed after a removal deleted that directory recreated it holding nothing but a manifest -- invisible to `list_vms`, failing to load at every start, answering "VM not found" to a second removal, and claiming that VM's netd interfaces against collection for as long as the VMM runs. The interfaces a netd outage kept the removal from releasing then had nothing left that could reach them. The refusal moves to the top, before the compose file is written, and the lock is taken there and held to the end, with the refusal repeated under it. `validate_port_mapping_nics` is gated the way the publishability check beside it already is. It ran on every update, against the node's current default, so changing `cvm.networking.nic.mode` made every later update of a VM that pinned a mapping fail over a field the request never touched -- the unmanageable- rather-than-fixed outcome the comment three lines below it rules out. * fix(vmm): stop a netd the VMM cannot open from reading as one that is not there Every `connect` failure carried the `Unreachable` marker, and both callers that treat absence as "nothing to do on this host" -- the interface release and the periodic collection -- skip their work at `debug!` when they see it. The default configuration reaches that state: netd runs as root and chmods its socket to `0660`, while the VMM is meant to run unprivileged, so a VMM whose user is not in root's group gets `EACCES` on every call. Interfaces then accumulate with nothing said at the default filter, while `create_vm` tells the operator to go and run a netd that is already running. Only the two errnos that mean nothing is listening are read as absence now. `hello` is answered before the blocking pool rather than inside it. The pool is finite and its tasks cannot be cancelled, so a node whose `virsh` calls are all timing out fills it, and a `hello` queued behind them times out too -- putting back the busy-netd-reads-as-absent this daemon exists to avoid. A sweep that deleted an interface but could not clear its nwfilter binding reported success, while the orphaned-binding half of the same loop treated the same failure as an error. It reports the pass as incomplete, which is what the stop it came from already knows how to say. `owner_of` strips what sysfs appended rather than trimming the whole alias: an identity with trailing whitespace re-derived a name that is not the one it is on, leaving the interface permanently unattributable -- never collected, and removable only by hand. * fix(vmm/ui): carry the NIC a port mapping was pinned to `normalizePorts` built the pin and then rebuilt the object without it in a trailing `map`, so the web UI could not pin a mapping at all -- and an edit made in the UI silently unpinned a mapping that had been pinned from the CLI, which the composable's own comment says cannot happen. TypeScript did not catch it because the field is optional. Three more, all in the same feature: - A cleared NIC box pinned NIC 0. `v-model.number` hands back the raw string when it does not parse, and `Number('')` is 0. - The update dialog never passed `nic-count`, so its NIC column was always hidden. A VM shrunk to one NIC could not have a stale pin cleared. - The update always sent `update_ports`, so the server-side "only when this request moved one of them" gate did not cover the UI at all: on a node whose netd does not forward host ports, a VM with any port mapping could not be edited from the UI in any respect. It is sent only when the mappings differ from what the dialog opened with. * docs: stop recommending the setuid helper this PR stopped using `setup-bridge.sh check` still failed the node when `qemu-bridge-helper` was not setuid root and `/etc/qemu/bridge.conf` had no `allow` line, three pages after the doc it is recommended by says neither is needed any more. Following both left a setuid binary nothing uses and a standing grant for any local user to attach a TAP to the bridge. The checks and the setup steps go; the teardown still removes the `allow` line, now saying why. Four statements corrected to match the code: - `dstack-vmm netd -c vmm.toml` does not parse. `--config` is not a global argument, so it has to come before the subcommand, as every other doc has it. - A VM on user networking does contact the netd socket: the release runs on every launch and every stop, before the decision about whether anything needs building. Nothing about the VM depends on the answer, which is the part worth saying. - The `instance_id` comment described two VMMs collecting each other's running VMs as a live hazard. A VMM that finds another live instance on its value refuses to start. - The CLI guide's port mapping section never mentioned `@`, which is in `--help` and in the onboarding doc. * refactor(vmm): stop tracking whether the host publishes a port `netd` in this repository builds interfaces; it does not forward host ports. `prepare_bridge` validates the `ingress` field of a request and then never reads it, and `capabilities()` said `ingress: false` unconditionally. Every mechanism built on top of that answer therefore described a forwarding netd that does not exist -- and each one had to be right about a host state the VMM does not control and cannot re-check, which is a bug surface bought with nothing. Gone, and with them 700 lines: - `PortMapping.published` in the RPC, `published_at`, `IngressBinding` and its `answers`, `Networking.ingress`, and the echo-and-compare in the launch path. No shipped client ever surfaced the field. - `refuse_unpublishable_ports`. It made `create_vm` with a port mapping fail outright on every bridge node running this repository's netd -- including `dstackup install`, which creates the KMS VM with one -- and refused on a netd that was merely restarting. What is left is the part that is a fact about the VM rather than about the host: a pin to a NIC that is macvtap or does not exist, or a VM with no user-mode and no bridge NIC at all. That check is local, synchronous, and now covers the unpinned case too, so `create_vm` and `update_vm` ask exactly one question about port mappings. - The `hello` handshake: `Capabilities`, `Reachability`, `probe`, the 30-second answer cache, `supports`, `records_ownership`, `forwards_ingress`. It existed to tell "netd does not have this operation" from "the operation failed", to decide between a sweep and a fallback. With the fallback gone there is nothing to decide: the release asks netd to sweep, and a refusal is a warning. What it holds is reclaimed by the VM's next launch or by reconciliation. - `release_recorded_interfaces` and `LEGACY_TEARDOWN_SPAN`, the eight-round-trip teardown for a netd too old to sweep. netd ships in this binary; the fix for one that cannot sweep is to restart it. `release_vm_interfaces` no longer takes the recorded networks -- it sweeps by identity, which is what made the record unnecessary -- so three call sites stop reading `runtime_networks` to hand it something it ignored, and `finish_remove_vm` loses one of the two `?`s that could strand a removal. * refactor(vmm): say that only QEMU publishes a host port, and stop pretending Three structures modelled a host that forwards bridge-NIC ports. Nothing on this host does. `PrepareBridgeRequest.ingress` and `IngressRequest` carried a per-NIC port list to netd, which read it exactly zero times. It was a declaration of intent to a daemon with no mechanism to honour it -- the request half of the `published` chain the previous commit removed from the response. With it gone, `mode_carries_ingress` can say what is true: QEMU's `hostfwd=`, and nothing else. That makes `default_ingress_nic` one line rather than a fallback to a bridge NIC that would have accepted a mapping and dropped it -- and a bridge pin is now refused at deployment, where the caller is there to be told, rather than resolving to a NIC with no path into the guest. The refusal is narrower in the other direction: an *unpinned* mapping is never refused, because a bridge-only VM with a port map deploys today and must keep deploying. What it strands, the launch names. `NetdInterface` and `netd_teardown` were a closed loop. At the base of this branch the record decided which per-NIC `Remove` teardown sent; this PR replaced that with `remove_all`, which derives names and needs no record. What was left wrote the field, persisted it, and read it only through `netd_teardown`, whose one production caller assigned the result straight back into the field it had just read. `is_filtered` had no callers at all. The 6-line comment justifying it still described the behaviour this PR deleted. `Request::Remove.filtered` was mandatory on decode and explicitly ignored by the handler, kept for "a netd that predates that reasoning". `origin/next` has no such field on `Remove`, so the compatibility was with a stacked branch rather than with anything released -- and serde ignores an unknown flattened key, so an in-flight old netd decodes a request without it either way. Also repairs two comments a `published: None` cleanup truncated mid-sentence, and three places -- a doc, a vmm.toml comment and a qemu.rs comment -- claiming netd "arbitrates host ports between VMM instances". * refactor(vmm): one notion of a failed sweep, not two and a bit `sweep_vm_interfaces` reported failure three ways. A binding it could not delete was an error in the branch for an orphaned one and a `warn!` plus an `incomplete` bit in the branch for a live one -- the same failure, read two ways, and the silent reading was the one that mattered. Both halves now fold into the same `first_error`, which folds the two branches into one loop and retires `remove_interface_in_pass`. That leaves `incomplete` with one producer: `COLLECTION_DEADLINE`, which cannot fire. The pass is 256 `stat()` calls; libvirt is asked once for a listing and at most once more for a delete, because the first failure latches it off. To spend twenty seconds it would need ~250 interfaces present for a single VM, against a `validate_identity` cap of 256 NICs and a real VM's one to four. The bit was threaded through `Outcome::Swept`, `Response`, `netd::Sweep`, a `warn!` and a `println!`, and no branch anywhere read it. `remove_all` returns a count. `InterfaceRecord.bound` goes the same way: written by `list_interfaces`, read only by the `FILTERED` column of `netd list`, where `kind` already says `binding` for the one case it distinguishes -- and it reported "no" both for an interface with no binding and for one libvirt could not be asked about. `create_vm` validated its port mapping NICs twice, on the same mappings against the same modes, because `create_manifest_from_vm_config` had already done it. Deleting the second call lets the UI drop `originalPorts` and the `JSON.stringify` deep-compare it needed to decide whether to send `update_ports`: a read-modify-write client does send the ports, so saying so is honest, and what the server checks on that path is now a local question about the VM's own pins. Also inlines `binding_cleanup` into its only caller, and corrects three comments the capability-probe removal left describing things the code no longer does. * refactor(vmm): retry a removal that could not release, instead of collecting after it The leak this PR set out to close is a VM removed while its interfaces could not be released: the release is deliberately non-fatal, so the removal went on to delete the workdir, and with it the only thing on the host that could still name what netd was holding. The answer was a whole-host reconciliation -- `netd list`, a claimable-VM set, a per-VM re-check under the launch lock, and an hourly task -- deciding from the outside which interfaces no VM claims. The VM already knows. `release_vm_interfaces` now says whether the sweep landed, and `finish_remove_vm` deletes the workdir only when it did. If netd refused, or was not there to ask, the directory and its `.removing` marker stay and the next VMM start resumes the removal, which `reload_vms` already knows how to do. `remove_all` is idempotent, so the retry costs one round trip. A VM that never asked netd for an interface is unaffected: the removal reads its recorded networks first, so an absent netd is not a reason to keep the directory of a VM netd never held anything for. Retrying at the next start is also strictly better than the timer it replaces, which waited up to an hour and skipped any VM whose launch lock was held. Gone with it: `reconcile_netd_interfaces`, `claimable_vm_ids`, `claims_vm`, `try_launch_lock`, `netd_reconcile_task`, `netd.reconcile_interval_secs`, and the startup pass -- about 500 lines including tests and docs. An existing `vmm.toml` that still sets the interval keeps loading; nothing denies unknown fields. Also gone is the duplicate-`instance_id` startup refusal and the `instance_id` it added to the discovery record. It caught one configuration and silently passed four -- a peer under a different uid (the discovery directory is per-user while the netd socket is host-wide), a peer registered by an older VMM, two VMMs starting at once, and any later edit -- while its error message taught operators that the collision was checked. `vmm.toml` says plainly that two instances must not share the value, and the derived default cannot collide. Kept: the ownership record. Teardown never needed it -- a sweep derives the names it deletes -- but an operator does, and on a host running several VMM instances it is the only thing that tells one instance's interfaces from another's. `netd list`, `remove-vm` and `remove-interface` are now the whole recovery story for what no VMM will retry: a workdir deleted by hand, or an interface recorded under an instance ID nothing uses any more. * docs(vmm): correct three comments the collection removal left behind * docs(netd): correct two comments the ingress removal left behind * docs(vmm): correct two more comments the ingress removal left behind --------- Co-authored-by: Kevin Wang --- docs/bridge-networking.md | 137 +- docs/libvirt-network-filter.md | 53 +- docs/network-data-plane.md | 51 +- docs/onboarding.md | 2 +- docs/vmm-cli-user-guide.md | 5 + dstack/crates/dstack-cli-core/src/ports.rs | 38 + dstack/crates/dstack-cli/src/main.rs | 3 +- dstack/crates/dstackup/src/install.rs | 3 + dstack/scripts/setup-bridge.sh | 112 +- dstack/vmm/rpc/proto/vmm_rpc.proto | 10 + dstack/vmm/src/app.rs | 690 ++++++--- dstack/vmm/src/app/network.rs | 502 +++---- dstack/vmm/src/app/qemu.rs | 151 +- dstack/vmm/src/app/vm_info.rs | 14 +- dstack/vmm/src/config.rs | 97 +- dstack/vmm/src/main.rs | 104 ++ dstack/vmm/src/main_service.rs | 174 ++- dstack/vmm/src/netd.rs | 1230 +++++++++++++++-- dstack/vmm/src/one_shot.rs | 50 +- dstack/vmm/src/vmm-cli.py | 28 +- .../vmm/ui/src/components/CreateVmDialog.ts | 5 +- .../ui/src/components/PortMappingEditor.ts | 17 + .../vmm/ui/src/components/UpdateVmDialog.ts | 5 +- dstack/vmm/ui/src/composables/useVmManager.ts | 30 +- dstack/vmm/vmm.toml | 29 +- 25 files changed, 2482 insertions(+), 1058 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 6bf3e6d5e..43216d086 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -143,35 +143,37 @@ mode = "bridge" bridge = "dstack-br0" ``` -### QEMU bridge helper setup (needed unless every bridge NIC goes through netd) +### netd is required -The bridge helper allows QEMU to create and attach TAP devices without VMM needing root privileges. -It is used only on the single-queue bridge paths; a NIC that `netd` builds never touches it, so a -node that runs `netd` for all of its bridge VMs does not need it at all. - -The VMM probes `/usr/lib/qemu/qemu-bridge-helper`, `/usr/libexec/qemu-bridge-helper` and -`/usr/local/libexec/qemu-bridge-helper`. Set `cvm.qemu_bridge_helper` in `vmm.toml` for a path -outside that list. +Bridge networking needs `netd`, the privileged helper that owns every host +interface a bridge or macvtap NIC uses. It is the same binary: ```bash -# Allow QEMU to use the bridge -sudo mkdir -p /etc/qemu -echo "allow virbr0" | sudo tee /etc/qemu/bridge.conf -# Or for manual bridge: echo "allow dstack-br0" | sudo tee /etc/qemu/bridge.conf - -# Set setuid on bridge helper -sudo chmod u+s /usr/lib/qemu/qemu-bridge-helper +sudo dstack-vmm --config vmm.toml netd ``` +Nothing else on the node needs `CAP_NET_ADMIN`: the VMM itself still runs +unprivileged, and `netd` holds the privilege behind a Unix socket whose +filesystem permissions authorize callers. + +This used to be conditional — `netd` built the TAP when libvirt filtering was on +or when the NIC wanted more than one queue pair, and otherwise QEMU's setuid +`qemu-bridge-helper` did. Two owners meant two answers to the same questions: +which netdev QEMU gets, whether vhost is really on, and what a bridge NIC's TAP +is built with. So a bridge NIC's host interface has one owner now, on every +node. + +`qemu-bridge-helper` is no longer used, and `/etc/qemu/bridge.conf` no longer +needs an `allow` line for the bridge. + ## How it works -- With more than one queue pair, or with libvirt filtering on, `netd` creates the TAP and the VMM passes `-netdev tap,id=net0,ifname=,...` — this is the usual case on a node running `netd` with multi-vCPU VMs, since queue pairs default to the VM's vCPU count. Without `netd`, a bridge NIC that took that default drops back to one queue pair and takes a helper path below -- Otherwise the VMM passes `-netdev tap,id=net0,br=,helper=,vhost=on`, or `-netdev bridge,id=net0,br=` when vhost is off or no helper is found -- QEMU's bridge helper (setuid) creates a TAP device and attaches it to the bridge on the two helper paths +- `netd` creates a persistent TAP, attaches it to the bridge, binds the nwfilter if the node filters, and the VMM passes `-netdev tap,id=net0,ifname=,...` - Guest MAC address is derived from SHA256 of the VM ID, with an optional configurable prefix (stable across restarts for DHCP IP consistency) - The host DHCP server (dnsmasq) assigns an IP to the VM -- On the two bridge-helper paths the TAP disappears when QEMU exits; a `netd`-created TAP is persistent and is deleted when the VMM tears the VM's networking down -- The VMM process itself needs neither root nor `CAP_NET_ADMIN` on any path; the `netd` path moves that privilege into a separate root service instead +- The TAP outlives QEMU and is deleted when the VMM tears the VM's networking down, so a VM that crashes does not leave its filter rules attached to a name the next VM could take +- Every interface `netd` creates records which VM of which VMM instance it belongs to, in the kernel's interface alias — see [Who owns an interface](#who-owns-an-interface) +- The VMM process needs neither root nor `CAP_NET_ADMIN`; `netd` holds that privilege in a separate service ### MAC address prefix @@ -199,6 +201,101 @@ The remaining bytes are derived from the VM ID hash. The prefix applies to all n - Docker's nftables chains (`DOCKER-FORWARD`) run before libvirt's but do not block virbr0 traffic - Use `setup-bridge.sh check --bridge ` to diagnose missing rules +### Which NIC a port mapping uses + +A port mapping says which NIC its traffic enters through: + +```bash +vmm-cli.py deploy ... --port udp:0.0.0.0:7483:51820@0 --port tcp:127.0.0.1:7484:8001@0 +``` + +Leave `@` off and the VMM picks the first user-mode NIC — where QEMU's +`hostfwd=` entries have always gone — and failing that the first bridge NIC. A +single-NIC VM never needs it. + +With several NICs the choice used to be made silently, and not always the way an +operator would have. A bridge NIC for external traffic beside a user-mode NIC for +management — the topology multi-NIC was added for — put every published port on +the *management* NIC: the traffic reached the guest, but over slirp, bypassing +whatever the bridge NIC's nwfilter was there to enforce and hiding the client's +address behind the slirp gateway. A second user-mode NIC could never publish +anything at all, because only the first was ever selected. + +A mapping resolves to exactly one NIC, and that NIC's backend decides the +mechanism: `hostfwd=` for user mode, `netd` for a bridge. Nothing can be claimed +by both. + +### Which ports a bridge NIC can publish + +QEMU publishes a port with `hostfwd=` on a user-mode NIC, and that is the only +mechanism this host has. **The `netd` in this repository builds interfaces; it +does not forward host ports**, so a bridge NIC cannot carry a port mapping. + +`--port …@` therefore only ever names a user-mode NIC. Pinning to a bridge, +macvtap or custom NIC is refused at deployment, where the caller is there to be +told. An unpinned mapping goes to the first user-mode NIC; a VM that has none is +not refused — it may have been deployed before this — but every mapping it +strands is named in the launch log. + +## Who owns an interface + +`netd` names an interface `dt<12 hex>`, a digest of (VMM instance, VM, NIC +index). That answers "where is this VM's interface" but not "whose is this +interface" — and the second question is the one a leaked interface poses. So +`netd` also records the identity on the interface itself: + +```console +$ ip -d link show dtc41d9e0b7a52 | grep alias + alias dstack1:0:path-3f9a1c8e7d2b4a60:0a1b2c3d4e5f6071 +``` + +The kernel holds that for exactly the interface's lifetime, so unlike a file on +disk it cannot be written late, lost, or left behind. It is a hint, never an +authority: a record is believed only when re-deriving the interface name from +it reproduces the name it is written on, so a forged, truncated or ambiguous +record reads the same as no record at all. + +Teardown does not need it — a sweep derives the names it deletes. What needs it +is an operator, and a host running several VMM instances, where it is the only +thing that tells one instance's interfaces from another's. + +```bash +# What netd holds on this host +sudo dstack-vmm netd list + +# Everything one VM holds, for a VM whose VMM will never ask again +sudo dstack-vmm netd remove-vm --instance path-3f9a1c8e7d2b4a60 --vm 0a1b2c3d4e5f6071 +``` + +### When a release does not land + +Every stop and every removal asks `netd` to sweep that VM's interfaces, by +deriving each of the 256 names its identity could produce. That needs no +record, and it reaches what a per-NIC teardown cannot: an interface a crash +left behind before anything on disk pointed at it, or one whose NIC the +manifest has since dropped. + +A removal deletes the VM's directory, and that directory — with its `.removing` +marker — is the only thing left that says to try again. So it is deleted only +once the sweep has landed. If `netd` refused, or was not there to ask, the +directory stays and the next VMM start resumes the removal; `remove_all` is +idempotent, so the retry costs one round trip. A VM that never asked `netd` for +an interface is unaffected: there is nothing for `netd` to be holding. + +What no VMM will retry is an interface whose VM directory an operator deleted +by hand, or one recorded under an instance ID no VMM uses any more. `netd list` +shows both, with the instance and VM they are recorded under: + +```bash +sudo dstack-vmm netd list +sudo dstack-vmm netd remove-vm --instance --vm +sudo dstack-vmm netd remove-interface dtc41d9e0b7a52 +``` + +Changing `cvm.instance_id` — or `run_path`, which it is derived from — strands +interfaces the same way. Running VMs keep working until they stop, and +`netd list` still shows the old instance ID, which is what `remove-vm` needs. + ### Mixing networking modes Bridge and user-mode VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index 6cc2724cf..3a1313aef 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -14,13 +14,17 @@ host mechanism. The measurable acceptance criteria are: -- `network_filter = "none"` installs no nwfilter binding. It still uses `netd` - for any NIC with more than one queue pair, and a `tap` netdev behind - `qemu-bridge-helper` whenever vhost is on; only a single-queue, non-vhost - bridge NIC keeps the historical `-netdev bridge` path with no `netd` or - libvirt dependency. +- `network_filter = "none"` installs no nwfilter binding. It does not remove + the `netd` dependency: `netd` creates the TAP for every bridge NIC either + way, and the VMM uses `-netdev tap` either way. What changes is only whether + that TAP carries a binding. - `network_filter = "libvirt"` creates the TAP and filter binding before QEMU is submitted to Supervisor, and uses QEMU `-netdev tap`. +- An nwfilter binding outlives the TAP it was bound to, so a teardown clears + the binding at every name that VM could have used, whether or not the + interface is still there. `dstack-vmm netd list` shows a binding whose + interface is already gone as a `binding` row; remove one with + `dstack-vmm netd remove-interface `. - A failed TAP or filter setup prevents QEMU from starting and rolls back all interfaces prepared for that VM. - Normal stop and removal delete the filter binding and TAP. @@ -105,6 +109,26 @@ arguments. It never accepts a command, executable path, TAP name, or raw XML from a client. Filter XML is generated internally with XML escaping and is validated by libvirt. +Teardown by identity only reaches the NIC indices its caller still has a record +of, and that record is written *after* the interface exists — a VMM killed in +between leaves a TAP nothing on disk points at, and a manifest that lost a NIC +leaves the same thing behind. `remove_all` names a VM instead of an interface +and derives every name that VM could occupy, so neither has to be recorded for +teardown to work. The VMM sweeps before preparing a launch as well as on stop, +which makes a launch self-healing regardless of what the record says. + +A bridge prepare also carries two things `netd` does not need to build the TAP. +`workdir` names the VM's directory on the host: untrusted, never read for a +decision, and present only so an operator reading `netd`'s log can get from an +opaque TAP name back to the VM. `ingress` states the host ports that NIC should make +reachable at its guest, which the VMM cannot arrange itself — it runs without +`CAP_NET_ADMIN` by design, and QEMU's `hostfwd=` entries need a user-mode netdev +that a bridge NIC does not have. The `netd` in this repository builds interfaces +and does not forward ports; it says so by leaving `ingress` out of its response, +the same reading `queues` gets, so a caller can tell "this netd does not do that" +from "nothing was asked for" instead of assuming ports were forwarded because a +TAP came back. + ## Deployment modes Production should run one shared service. `netd` reads the `[netd]` section, @@ -194,11 +218,11 @@ sudo dstack-vmm --config ./vmm.toml \ --netd-socket /run/dstack-dev/netd.sock ``` -User networking never asks `netd` to build an interface; the VMM still opens a -short liveness-probe connection to the netd socket on every launch and when -describing a stopped VM. Libvirt mode fails closed if `netd` -is unavailable. Bridge networking with `mode = "none"` connects only when it -needs more than one queue pair, as described below. +User networking and a caller-supplied netdev never ask `netd` to build an +interface. The VMM still contacts the socket for such a VM -- every launch and +every stop releases whatever the VM held, before it decides whether it needs +anything built -- but nothing about the VM depends on the answer. Bridge and +macvtap do ask, and fail closed if `netd` is unavailable. Filtered TAP netdevs follow the node's `vhost` and `queues` settings like any other TAP-backed NIC (see [network-data-plane.md](network-data-plane.md)). The @@ -207,10 +231,11 @@ whether they were written by QEMU or by a vhost worker; filtering is unaffected by the data plane choice. Enabling vhost does require the QEMU user to be able to open `/dev/vhost-net`. -`netd` also creates the TAP for unfiltered bridge NICs that ask for more than -one queue pair, because `qemu-bridge-helper` returns a single descriptor and -cannot create a `multi_queue` device. Those TAPs carry no nwfilter binding, so -a multiqueue bridge node needs `netd` even when `network_filter.mode = "none"`. +`netd` creates the TAP for unfiltered bridge NICs too. Those TAPs carry no +nwfilter binding, so a bridge node needs `netd` even when +`network_filter.mode = "none"` — see +[bridge-networking.md](bridge-networking.md) for why the host interface has a +single owner. An empty filter name is what selects that unfiltered TAP, so `mode = "libvirt"` with an empty `filter` is rejected at config load rather than quietly producing diff --git a/docs/network-data-plane.md b/docs/network-data-plane.md index 57f0134a1..c80af880f 100644 --- a/docs/network-data-plane.md +++ b/docs/network-data-plane.md @@ -64,8 +64,8 @@ refuses a request for four queue pairs should not hand out sixteen by itself. The hard ceiling from any source is 64. Without vhost the default is a single queue pair. The QEMU main loop drains -every queue on one thread, so extra queues buy little while still costing a -netd interface, more MSI-X vectors, and a changed guest device. An explicit +every queue on one thread, so extra queues buy little while still costing more +MSI-X vectors and a changed guest device. An explicit queue count is still honoured without vhost, since that combination is a deliberate request rather than a default. The two defaults travelling together also means a node that never sets `vhost` keeps building the device its VMs @@ -150,9 +150,9 @@ measured, so attestation and app identity are unaffected. Before flipping it: restarting. The VMM warns at startup when its own access fails, but it cannot refuse on that basis — QEMU need not share its credentials. -2. **Restart `netd` before or together with the VMM.** Multiqueue bridge NICs - are prepared by `netd`, and the VMM checks that `netd` echoes the queue - count it built. An older `netd` fails that check; the launch is rolled back +2. **Restart `netd` before or together with the VMM.** Every bridge and + macvtap NIC is prepared by `netd`, and the VMM checks that `netd` echoes the + queue count it built. An older `netd` fails that check; the launch is rolled back and fails with the reason in the VMM log, but the VM does not start until `netd` is upgraded. @@ -165,35 +165,22 @@ measured, so attestation and app identity are unaffected. Before flipping it: | Mode | netdev | vhost | queues > 1 | |---|---|---|---| | `user` | `user,...` | no backend | not supported | -| `bridge` | `tap,ifname=` via netd, else `tap,br=,helper=`, else `bridge,br=` | yes | yes, through netd | -| `bridge` with libvirt filtering | `tap,ifname=` | yes | yes, through netd | +| `bridge` | `tap,ifname=` via netd | yes | yes | | `macvtap` | `tap,fd=` / `tap,fds=` | yes | yes | | `custom` | operator's own string | operator's own string | no, not settable | -QEMU's `bridge` netdev accepts neither `vhost=` nor `queues=`, so enabling -vhost switches bridge mode to a `tap` netdev driven by the same setuid -`qemu-bridge-helper`. The VMM still needs no network privileges. The helper has -no compiled-in default path for the `tap` netdev, so the VMM probes the known -distribution locations; set `cvm.qemu_bridge_helper` if yours is elsewhere. If -no helper is found the NIC falls back to the non-vhost `bridge` netdev with a -warning, because a node-wide setting must not stop a node from booting VMs -that never asked for it. - -The helper returns exactly one descriptor, which is why more than one queue -pair in bridge mode is created by `netd` instead: it adds a persistent -`multi_queue` TAP that QEMU then opens once per queue. `netd` requires the -`virsh` binary to be installed even when nothing is filtered, though it does -not require a reachable `libvirtd`. That applies whether or -not libvirt filtering is on, so a bridge node needs `netd` to get the default -queue count (see [libvirt-network-filter.md](libvirt-network-filter.md)). -Without it, bridge NICs fall back to a single queue pair with a warning rather -than failing to launch; a VM that asked for a queue count explicitly still -fails, so the caller learns their request was not met. `netd` is probed by -connecting, not by looking for its socket file, because a `netd` that died -leaves the socket behind. One-shot `dstack-vmm run` has no netd lifecycle at -all and behaves like a node without it. `netd` reports back the -queue count it created, and the VMM refuses to launch on a mismatch — a `netd` -deployed separately as a root service can be older than the VMM asking it for +QEMU's `bridge` netdev accepts neither `vhost=` nor `queues=`, and the setuid +`qemu-bridge-helper` behind its `tap` netdev returns exactly one descriptor. So +bridge mode runs on a TAP that `netd` creates: persistent, `multi_queue` when +asked for, and opened once per queue by QEMU. That is true of every bridge NIC, +filtered or not, single-queue or not — see +[bridge-networking.md](bridge-networking.md) for why the host interface has one +owner. `netd` requires the `virsh` binary to be installed even when nothing is +filtered, though it does not require a reachable `libvirtd`. One-shot +`dstack-vmm run` does not manage netd interface lifecycle, so it refuses bridge +and macvtap NICs outside `--dry-run`. `netd` reports back the queue count it +created, and the VMM refuses to launch on a mismatch — a `netd` deployed +separately as a root service can be older than the VMM asking it for multiqueue, and QEMU would otherwise reject the interface from inside the per-VM launcher. @@ -253,7 +240,7 @@ multiqueue line rate with zero `swiotlb buffer is full` events. `vectors` is derived, never configured: `2N + 2`, one vector per queue direction plus config and control. One queue pair emits no `mq=on` or `vectors=` at all, leaving the guest device line byte for byte identical to the -one before this feature. The `-netdev` half does change wherever vhost is on, +one before this feature. The `-netdev` half carries `vhost=on|off` either way, since that is what selects the backend. ## Requirements diff --git a/docs/onboarding.md b/docs/onboarding.md index 0d4488bab..28b216ff2 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -149,7 +149,7 @@ The deploy command: Pass `--vcpu`, `--memory`, or `--disk` to change the app resources before you deploy. -The `--port 8080:80` mapping means `host_port:vm_port` and uses TCP on `127.0.0.1`. The full accepted forms are `vm`, `host:vm`, `proto:host:vm`, and `proto:addr:host:vm`. Use `tcp` or `udp` for `proto`. Fixed host and VM ports must be between 1 and 65535. If you omit the host port, or use `auto` or `0`, `dstack` picks a free localhost port and prints the selected mapping after deploy. +The `--port 8080:80` mapping means `host_port:vm_port` and uses TCP on `127.0.0.1`. The full accepted forms are `vm`, `host:vm`, `proto:host:vm`, and `proto:addr:host:vm`, each optionally suffixed with `@` to name which NIC the traffic enters through (a single-NIC VM never needs it). Use `tcp` or `udp` for `proto`. Fixed host and VM ports must be between 1 and 65535. If you omit the host port, or use `auto` or `0`, `dstack` picks a free localhost port and prints the selected mapping after deploy. Open the app from the host: diff --git a/docs/vmm-cli-user-guide.md b/docs/vmm-cli-user-guide.md index 6a669ca4e..1f803e31c 100644 --- a/docs/vmm-cli-user-guide.md +++ b/docs/vmm-cli-user-guide.md @@ -290,6 +290,11 @@ Expose services running in your VM: # Multiple ports --port tcp:8080:80 --port tcp:8443:443 + +# Pin a mapping to one NIC: protocol[:host_address]:host_port:vm_port@ +# Without @ the mapping enters through the first user-mode NIC, or the +# first bridge NIC when the VM has no user-mode one. +--port tcp:0.0.0.0:8443:443@0 ``` #### GPU Assignment diff --git a/dstack/crates/dstack-cli-core/src/ports.rs b/dstack/crates/dstack-cli-core/src/ports.rs index ae662442a..142be1b16 100644 --- a/dstack/crates/dstack-cli-core/src/ports.rs +++ b/dstack/crates/dstack-cli-core/src/ports.rs @@ -33,7 +33,25 @@ pub fn tcp_port_free(addr: &str, port: u16) -> bool { /// * `:` — tcp, 127.0.0.1 /// * `::` /// * `:::` +/// +/// Any of them may carry a trailing `@` naming which NIC the traffic +/// enters through, as `vmm-cli.py --port` does. Without it the VMM picks: the +/// first user-mode NIC, else the first bridge NIC. A single-NIC VM never needs +/// it. pub fn parse_port(spec: &str) -> Result { + let (spec, nic_index) = match spec.rsplit_once('@') { + // Only digits. `parse()` would take " 1" and "+1" as 1, and a NIC index + // is a position in a list the caller wrote. + Some((rest, nic)) if nic.bytes().all(|byte| byte.is_ascii_digit()) && !nic.is_empty() => ( + rest, + Some( + nic.parse::() + .with_context(|| format!("invalid NIC index in --port '{spec}'"))?, + ), + ), + Some((_, nic)) => bail!("invalid NIC index in --port '{spec}': '{nic}' is not a number"), + None => (spec, None), + }; let parts: Vec<&str> = spec.split(':').collect(); let (proto, addr, host, vm) = match parts.as_slice() { [vm] => ("tcp", "127.0.0.1", "auto", *vm), @@ -58,6 +76,7 @@ pub fn parse_port(spec: &str) -> Result { host_address: addr.to_string(), host_port, vm_port, + nic_index, }) } @@ -90,4 +109,23 @@ mod tests { assert!(parse_port("70000:80").is_err()); assert!(parse_port("8080:0").is_err()); } + + #[test] + fn a_mapping_can_name_the_nic_it_enters_through() { + assert_eq!(parse_port("8080:80").unwrap().nic_index, None); + + let pinned = parse_port("udp:0.0.0.0:7483:51820@1").unwrap(); + assert_eq!(pinned.nic_index, Some(1)); + assert_eq!(pinned.protocol, "udp"); + assert_eq!(pinned.host_address, "0.0.0.0"); + assert_eq!(pinned.host_port, 7483); + assert_eq!(pinned.vm_port, 51820); + + // Unpinned must stay unpinned rather than default to NIC 0: the VMM + // resolves it to the first user-mode NIC, which need not be the first. + assert_eq!(parse_port("8080:80@0").unwrap().nic_index, Some(0)); + for bad in ["8080:80@", "8080:80@ 1", "8080:80@+1", "8080:80@a"] { + assert!(parse_port(bad).is_err(), "{bad} must be refused"); + } + } } diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 9624c3a27..8a70141e9 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -102,7 +102,8 @@ enum Command { /// disk size in GB. #[arg(long, default_value_t = 20)] disk: u32, - /// expose a port: `vm` | `host:vm` | `proto:host:vm` | `proto:addr:host:vm` + /// expose a port: `vm` | `host:vm` | `proto:host:vm` | `proto:addr:host:vm`, + /// each optionally suffixed `@` to name the NIC it enters through /// (host omitted/`auto`/`0` ⇒ a free host port is picked). Repeatable. #[arg(long = "port", value_name = "SPEC")] ports: Vec, diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index 6599bc3cb..b32f9fea0 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -294,6 +294,9 @@ pub(crate) async fn cmd_install(mut o: InstallOpts, release_api_base_url: &str) host_address: "127.0.0.1".into(), host_port: kms_port as u32, vm_port: 8000, + // Unpinned: this deploys the node default topology, which + // is one NIC, and the VMM resolves that itself. + nic_index: None, }], ..Default::default() }; diff --git a/dstack/scripts/setup-bridge.sh b/dstack/scripts/setup-bridge.sh index edcbe35a7..ecc3d2901 100755 --- a/dstack/scripts/setup-bridge.sh +++ b/dstack/scripts/setup-bridge.sh @@ -44,83 +44,6 @@ run_cmd() { fi } -# --- Detect qemu-bridge-helper path --- - -find_bridge_helper() { - local paths=( - /usr/lib/qemu/qemu-bridge-helper - /usr/libexec/qemu-bridge-helper - /usr/local/lib/qemu/qemu-bridge-helper - /usr/local/libexec/qemu-bridge-helper - ) - for p in "${paths[@]}"; do - if [[ -f "$p" ]]; then - echo "$p" - return 0 - fi - done - return 1 -} - -# --- Detect current bridge provider --- - -# Returns "libvirt:" if bridge is managed by a libvirt network, -# "standalone" otherwise. -detect_bridge_provider() { - if command -v virsh &>/dev/null; then - local name br - while read -r name; do - [[ -z "$name" ]] && continue - br=$(virsh net-dumpxml "$name" 2>/dev/null | grep -oP "/dev/null) - fi - echo "standalone" -} - -# --- Check functions --- - -check_bridge_helper() { - echo - bold "qemu-bridge-helper" - local helper - if ! helper=$(find_bridge_helper); then - check_fail "qemu-bridge-helper not found" - check_info "Install QEMU: sudo apt install qemu-system-x86" - return - fi - check_pass "found at $helper" - - if [[ -u "$helper" ]]; then - check_pass "setuid bit is set" - else - check_fail "setuid bit not set" - check_info "Fix: sudo chmod u+s $helper" - fi -} - -check_bridge_conf() { - echo - bold "/etc/qemu/bridge.conf" - local conf="/etc/qemu/bridge.conf" - if [[ ! -f "$conf" ]]; then - check_fail "$conf does not exist" - check_info "Fix: sudo mkdir -p /etc/qemu && echo 'allow $BRIDGE' | sudo tee $conf" - return - fi - check_pass "$conf exists" - - if grep -qE "^allow[[:space:]]+($BRIDGE|all)[[:space:]]*$" "$conf" 2>/dev/null; then - check_pass "bridge '$BRIDGE' is allowed" - else - check_fail "bridge '$BRIDGE' not found in $conf" - check_info "Fix: echo 'allow $BRIDGE' | sudo tee -a $conf" - fi -} - check_bridge_interface() { echo bold "bridge interface: $BRIDGE" @@ -326,33 +249,6 @@ check_forward_rules() { # --- Setup: common --- -setup_bridge_conf() { - echo - bold "Setting up /etc/qemu/bridge.conf" - run_cmd sudo mkdir -p /etc/qemu - if [[ -f /etc/qemu/bridge.conf ]] && grep -qE "^allow[[:space:]]+($BRIDGE|all)" /etc/qemu/bridge.conf 2>/dev/null; then - echo " already configured" - else - run_cmd bash -c "echo 'allow $BRIDGE' | sudo tee -a /etc/qemu/bridge.conf" - fi -} - -setup_bridge_helper() { - echo - bold "Setting up qemu-bridge-helper" - local helper - if ! helper=$(find_bridge_helper); then - echo " $(red 'ERROR'): qemu-bridge-helper not found. Install QEMU first." - return 1 - fi - if [[ -u "$helper" ]]; then - echo " setuid already set on $helper" - else - run_cmd sudo chmod u+s "$helper" - echo " setuid set on $helper" - fi -} - setup_ip_forward() { echo bold "Enabling IP forwarding" @@ -681,8 +577,6 @@ cmd_check() { echo "provider: $(bold 'standalone')" fi - check_bridge_helper - check_bridge_conf check_bridge_interface check_dhcp check_dhcp_firewall @@ -728,8 +622,6 @@ cmd_setup() { $DRY_RUN && echo "dry-run: $(yellow 'yes')" # Common setup - setup_bridge_conf - setup_bridge_helper setup_ip_forward # Mode-specific setup @@ -812,7 +704,9 @@ cmd_destroy() { fi fi - # Remove bridge.conf entry + # Remove the bridge.conf entry an older setup added. netd owns every host + # interface now, so qemu-bridge-helper is not used and the `allow` line is + # a standing grant to attach any local user's TAP to the bridge. local conf="/etc/qemu/bridge.conf" if [[ -f "$conf" ]] && grep -qE "^allow[[:space:]]+${BRIDGE}[[:space:]]*$" "$conf" 2>/dev/null; then echo diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 31fef51f4..62bf381be 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -182,6 +182,16 @@ message PortMapping { uint32 vm_port = 3; // Host address string host_address = 4; + // Which NIC this mapping's traffic enters through, as an index into + // `networks`. Unset picks the first user-mode NIC, which is where QEMU's + // hostfwd entries have always gone and the only place a host port is + // published from. + // + // A VM with one NIC never needs it. With several there is a choice, and it + // used to be made silently: a bridge NIC for external traffic beside a + // user-mode NIC for management -- the topology multi-NIC was added for -- + // put every published port on the management NIC. + optional uint32 nic_index = 5; } // Partial configuration used when mutating an existing VM. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index e9205b07d..f6deaa8ec 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -3,10 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - config::{ - Config, NetdInterface, Networking, NetworkingMode, NicNetworking, ProcessAnnotation, - Protocol, - }, + config::{Config, Networking, NetworkingMode, NicNetworking, ProcessAnnotation, Protocol}, logrotate, netd::{ self, InterfaceIdentity, PrepareBridgeRequest, PrepareMacvtapRequest, @@ -35,7 +32,6 @@ use rand::seq::SliceRandom; use serde::{Deserialize, Serialize}; use serde_json::json; use sha2::{Digest, Sha256}; -use std::cell::OnceCell; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use std::net::IpAddr; use std::path::{Path, PathBuf}; @@ -46,8 +42,8 @@ use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; pub(crate) use network::{ - clamp_queues_without_netd, filters_bridge_traffic, needs_netd_interface, netd_available, - netd_teardown, resolve_networking, resolved_networks, settle_vhost, validate_resolved_network, + filters_bridge_traffic, mode_carries_ingress, needs_netd_interface, resolve_networking, + resolved_networks, settle_vhost, stranded_ingress, validate_resolved_network, validate_resolved_networks, }; pub use qemu::VmConfig; @@ -61,7 +57,7 @@ mod host_share; mod id_pool; mod image; mod mr_config; -mod network; +pub(crate) mod network; mod qemu; pub(crate) mod registry; mod vm_info; @@ -97,6 +93,10 @@ pub struct PortMapping { pub protocol: Protocol, pub from: u16, pub to: u16, + /// Which NIC carries this mapping. `None` resolves by the node's rule; see + /// [`crate::app::network::ingress_nic`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nic_index: Option, } /// An extra disk attached to the VM (e.g. a pre-baked verity volume). `source` @@ -310,6 +310,9 @@ pub struct App { state: Arc>, /// Pull status for registry images: tag → status. pub(crate) pull_status: Arc>>, + /// One lock per VM, held across a launch or a teardown. See + /// [`App::launch_lock`]. + launch_locks: Arc>>>>, } const GUEST_AGENT_RPC_TIMEOUT: Duration = Duration::from_secs(30); @@ -337,12 +340,42 @@ impl App { state: Arc::new(Mutex::new(AppState { cid_pool, vms: HashMap::new(), + removing: HashSet::new(), })), config: Arc::new(config), pull_status: Arc::new(Mutex::new(std::collections::HashMap::new())), + launch_locks: Arc::new(Mutex::new(HashMap::new())), } } + /// Serializes everything that creates or deletes one VM's host interfaces. + /// + /// A launch reads whether QEMU is already up and then spends many awaits -- + /// a GPU reset, a whole netd conversation, building the QEMU arguments -- + /// before it launches anything. Nothing used to cover that window. Two + /// `StartVm` calls, or one racing the auto-restart timer, could both read + /// "not running", and the loser's sweep would delete the TAPs the winner's + /// QEMU was already holding open: a live VM silently loses its networking, + /// and the loser's error path then clears the winner's record of it. The + /// authoritative rejection lives in the supervisor, which is reached long + /// after the damage is done. + /// + /// A tokio mutex, because it is held across awaits. Per VM, because a slow + /// start must not stall unrelated ones. Taken by stop and by removal as + /// well as by start: those delete the same interfaces from the other side. + pub(crate) async fn launch_lock(&self, id: &str) -> tokio::sync::OwnedMutexGuard<()> { + self.launch_lock_handle(id).lock_owned().await + } + + fn launch_lock_handle(&self, id: &str) -> Arc> { + let mut locks = self.launch_locks.lock().or_panic("mutex poisoned"); + // A VM that is neither starting nor stopping leaves the map holding + // the only reference, so the map stays the size of what is in flight + // rather than of every VM this process has ever touched. + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + locks.entry(id.to_string()).or_default().clone() + } + pub async fn load_vm( &self, work_dir: impl AsRef, @@ -406,6 +439,19 @@ impl App { Ok(()) } + /// Refuses an operation on a VM that is being removed. + /// + /// Cheap, and taken before the launch lock as well as under it. Removal + /// holds that lock until the VM has exited -- hours, by its own estimate -- + /// so anything that only asked afterwards would wait the removal out in + /// order to be told no. + pub(crate) fn refuse_if_removing(&self, id: &str) -> Result<()> { + if self.lock().is_removing(id) { + bail!("VM is being removed"); + } + Ok(()) + } + pub async fn start_vm(&self, id: &str) -> Result<()> { self.start_vm_with_restart_policy(id, true).await } @@ -420,13 +466,24 @@ impl App { vm.state.auto_restart.reset(); } } - { - let state = self.lock(); - if let Some(vm) = state.get(id) { - if vm.state.removing { - bail!("VM is being removed"); - } - } + // Before the lock as well as after it. Removal holds the lock until the + // VM has exited, so a launch that only asked afterwards would wait that + // out -- hours, by removal's own estimate -- to be told no. Asking + // first is not sufficient on its own, because the marker can be set + // while this waits; asking again under the lock is what makes it + // authoritative. + self.refuse_if_removing(id)?; + // Everything below reads whether this VM is running and acts on the + // answer for as long as the launch takes. See [`App::launch_lock`]. + let _launch = self.launch_lock(id).await; + self.refuse_if_removing(id)?; + // A restart decided before a stop must not outlive it. The decision + // read `started` from disk; `stop_vm` writes it false under this lock, + // so re-reading it here is what makes the stop stick. An explicit start + // sets the flag itself and has nothing to re-read. + if !reset_restart_policy && !self.work_dir(id)?.started().unwrap_or(false) { + debug!(id, "skipping automatic restart: the VM was stopped"); + return Ok(()); } self.sync_dynamic_config(id)?; let is_running = self @@ -488,16 +545,12 @@ impl App { ) { Ok(processes) => processes, Err(error) => { - let _ = self - .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) - .await; + self.release_vm_interfaces(&vm_config.manifest.id).await; return Err(error); } }; if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) { - let _ = self - .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) - .await; + self.release_vm_interfaces(&vm_config.manifest.id).await; return Err(error); } { @@ -507,12 +560,7 @@ impl App { } for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { - if let Err(cleanup_error) = self - .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) - .await - { - warn!(id, %cleanup_error, "failed to roll back filtered networking"); - } + self.release_vm_interfaces(&vm_config.manifest.id).await; if let Err(clear_err) = work_dir.clear_runtime_networks() { warn!( id, @@ -542,13 +590,25 @@ impl App { } pub async fn stop_vm(&self, id: &str) -> Result<()> { + // Removal stops the VM itself and holds the launch lock while it does, + // so this would otherwise wait hours to do again what is already being + // done. + self.refuse_if_removing(id)?; if let Some(vm) = self.lock().get_mut(id) { vm.state.auto_restart.reset(); } + // Teardown deletes the same interfaces a launch creates, and derives + // their names rather than reading a record, so it must not overlap one. + let _launch = self.launch_lock(id).await; self.set_started(id, false)?; self.stop_vm_process(id).await?; - let networks = self.work_dir(id)?.runtime_networks(); - self.remove_filtered_networks(id, &networks).await?; + // Not fallible: a VM that has been asked to stop is stopped whether or + // not netd could be reached. What is left behind is reclaimed by this + // VM's next launch, which releases before it prepares, or by its + // removal, which will not finish until the release lands. A VM that is + // never started or removed again keeps its interfaces, and its + // directory is still there to say whose they are. + self.release_vm_interfaces(id).await; Ok(()) } @@ -557,16 +617,51 @@ impl App { vm: &VmConfig, networks: &mut [Networking], ) -> Result<()> { - if !networks - .iter() - .any(|network| needs_netd_interface(network, &self.config.cvm)) - { + // Before the early return, because a mapping with nowhere to go is a + // property of the resolved topology and not of whether netd is in it. + // Deployment refuses every way of asking for one, so reaching this + // means an edit removed the NIC out from under a mapping that named it. + for mapping in stranded_ingress(&vm.manifest.port_map, networks) { + warn!( + vm_id = %vm.manifest.id, + "port mapping {} {}:{} names NIC {:?}, which this VM no longer has a backend \ + for; it will not be published", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.nic_index, + ); + } + // Whatever an earlier boot left behind: a crash between creating an + // interface and recording it, a NIC this VM no longer has, or a whole + // backend it no longer uses. Prepare replaces the names it is about to + // use, but only those, so an index nothing will claim again is only + // reachable from here. + // + // Before the early return, and not inside it. A VM that has moved from + // a bridge to user-mode networking needs this release precisely because + // it no longer wants an interface, and gating it on wanting one is how + // the interfaces it left behind would become unreachable to every + // later launch. + self.release_vm_interfaces(&vm.manifest.id).await; + if !networks.iter().any(needs_netd_interface) { return Ok(()); } let qemu_uid = Uid::effective().as_raw(); + // Only ever read back out of a log line: netd is told where the VM + // lives so an operator holding an opaque TAP name can reach the VM + // without going through the VMM first. + let workdir = self + .work_dir(&vm.manifest.id) + .map(|dir| dir.path().display().to_string()) + .unwrap_or_default(); + // `port_map` is implemented as QEMU `hostfwd=` entries on a user-mode + // netdev, so a bridge NIC drops every one of them. The VMM cannot + // forward them itself -- it runs without CAP_NET_ADMIN by design -- so + // it states the requirement and lets the node's netd answer it. let mut prepared = Vec::new(); for (nic_index, network) in networks.iter_mut().enumerate() { - if !needs_netd_interface(network, &self.config.cvm) { + if !needs_netd_interface(network) { continue; } let identity = InterfaceIdentity { @@ -593,6 +688,7 @@ impl App { // libvirt at all. filtered, queues, + workdir: workdir.clone(), }), NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap(PrepareMacvtapRequest { identity: identity.clone(), @@ -601,20 +697,22 @@ impl App { qemu_uid, mode: network.macvtap_mode.clone(), queues, + workdir: workdir.clone(), }), NetworkingMode::User | NetworkingMode::Custom => continue, }; let response = match netd::request(&self.config.netd.socket, &request).await { Ok(response) => response, Err(error) => { - // The client may have timed out while netd was still finishing - // this Prepare. Remove the in-flight identity first; netd's - // serialized accept loop processes it after Prepare completes. + // The client may have timed out while netd was still + // finishing this Prepare. Remove the in-flight identity + // first: the operation lock makes netd run that removal + // after the Prepare it is undoing, whatever order the two + // connections arrived in. if let Err(cleanup_error) = netd::request( &self.config.netd.socket, &NetdRequest::Remove { identity: identity.clone(), - filtered, }, ) .await @@ -622,16 +720,17 @@ impl App { warn!(%cleanup_error, "failed to roll back in-flight filtered network"); } self.roll_back_prepared_networks(prepared).await; - // netd's own message is about a TAP, not about queues, so - // a caller who asked for multiqueue would not see their - // request named anywhere in the failure. + // netd's own message is about a socket or a TAP, so + // neither the NIC that asked nor what a node has to install + // to satisfy it appears anywhere in the failure. let unreachable = netd::is_unreachable(&error); + let mode = network.nic.mode.as_str(); let error = Err(error).context("failed to prepare netd-managed networking"); - return if queues > 1 && unreachable { + return if unreachable { error.with_context(|| { format!( - "interface {nic_index} asked for {queues} queue pairs, which needs \ - a netd running on this host" + "interface {nic_index} is {mode}, whose host interface only netd \ + can build; run dstack-vmm netd on this host" ) }) } else if queues > 1 { @@ -639,19 +738,11 @@ impl App { format!("interface {nic_index} asked for {queues} queue pairs") }) } else { - error + error.with_context(|| format!("interface {nic_index} is {mode}")) }; } }; - prepared.push((identity.clone(), filtered)); - // netd built this one. Record it now, before anything else can - // fail, so teardown never has to re-derive it from a node - // configuration the operator may since have changed. - network.netd_interface = if filtered { - NetdInterface::Filtered - } else { - NetdInterface::Unfiltered - }; + prepared.push(identity.clone()); // Everything below runs after netd already built a host interface, // so a failure has to unwind the same way a failed Prepare does. let accepted = (|| { @@ -692,45 +783,18 @@ impl App { /// down the snapshot describes a boot that is over: the node configuration /// and the VM's own manifest can both have changed since, so reporting it /// would answer a question about the past with the grammar of the present. - /// Predict instead, the same way the next launch will -- including the - /// drop to a single queue pair on a node with no netd. - /// - /// `netd_reachable` is shared across a request rather than probed here: - /// the probe is a blocking connect that netd's serialized accept loop has - /// to service, and one status query covers many VMs. - fn effective_networks( - &self, - info: &vm_info::VmInfo, - netd_reachable: &OnceCell, - ) -> Vec { + /// Predict instead, the same way the next launch will. + fn effective_networks(&self, info: &vm_info::VmInfo) -> Vec { if info.running && !info.runtime_networks.is_empty() { return info.runtime_networks.clone(); } - let available = *netd_reachable.get_or_init(|| netd_available(&self.config.netd.socket)); - self.merge_networks(&info.manifest, available).0 + self.merge_networks(&info.manifest) } - /// Launch-time view of a VM's NICs: node defaults merged in, the - /// vCPU-scaled queue count made concrete, and multiqueue dropped when this - /// node has no netd to build the interface. + /// Launch-time view of a VM's NICs: node defaults merged in and the + /// vCPU-scaled queue count made concrete. pub(crate) fn runtime_networks(&self, manifest: &Manifest) -> Vec { - let available = netd_available(&self.config.netd.socket); - let (networks, clamped, vhost_denied) = self.merge_networks(manifest, available); - if clamped > 0 { - warn!( - id = %manifest.id, - "netd is not available, so {clamped} bridge interface(s) fall back to a single \ - queue pair; run dstack-vmm netd to let queue pairs scale with vCPUs" - ); - } - if vhost_denied > 0 { - warn!( - id = %manifest.id, - "no qemu-bridge-helper found, so {vhost_denied} bridge interface(s) fall back to \ - the non-vhost bridge netdev; set cvm.qemu_bridge_helper to enable vhost" - ); - } - networks + self.merge_networks(manifest) } /// A running VM whose snapshot is missing, because a VMM that predates the @@ -747,93 +811,80 @@ impl App { /// queue pair and no vhost. Asking `runtime_networks` would apply today's /// defaults to a launch that predates them, and the guess is persisted, so /// it would keep describing that VM wrongly for the life of its boot. - /// - /// `merge_networks` rather than `runtime_networks` for the same reason: the - /// latter probes netd and warns about a multiqueue fallback, which says - /// nothing about a VM that is already up. fn inferred_runtime_networks(&self, manifest: &Manifest) -> Vec { - let mut networks = self.merge_networks(manifest, false).0; + let mut networks = self.merge_networks(manifest); for network in &mut networks { network.nic.vhost = Some(false); network.nic.queues = Some(1); } - for network in &mut networks { - network.netd_interface = match netd_teardown(network, &self.config.cvm) { - Some(true) => NetdInterface::Filtered, - Some(false) => NetdInterface::Unfiltered, - None => NetdInterface::None, - }; - } networks } - /// The merge itself, without the launch-time logging, plus how many NICs - /// lost multiqueue for want of netd. - fn merge_networks( - &self, - manifest: &Manifest, - netd_reachable: bool, - ) -> (Vec, usize, usize) { - let requested = if manifest.networks.is_empty() { - vec![self.config.cvm.networking.nic.clone()] - } else { - manifest.networks.clone() - }; + /// The merge itself: node defaults applied, then the data plane settled so + /// that every later stage reads one answer instead of recomputing it. + fn merge_networks(&self, manifest: &Manifest) -> Vec { let mut resolved = resolved_networks(manifest, &self.config.cvm); - let clamped = - clamp_queues_without_netd(&requested, &mut resolved, &self.config.cvm, netd_reachable); - let vhost_denied = settle_vhost(&mut resolved, &self.config.cvm); - (resolved, clamped, vhost_denied) + settle_vhost(&mut resolved); + resolved } /// Removes interfaces netd already built for a launch that then failed. - async fn roll_back_prepared_networks(&self, prepared: Vec<(InterfaceIdentity, bool)>) { - for (identity, filtered) in prepared.into_iter().rev() { - if let Err(cleanup_error) = netd::request( - &self.config.netd.socket, - &NetdRequest::Remove { identity, filtered }, - ) - .await + async fn roll_back_prepared_networks(&self, prepared: Vec) { + for identity in prepared.into_iter().rev() { + if let Err(cleanup_error) = + netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity }).await { warn!(%cleanup_error, "failed to roll back prepared network interface"); } } } - pub(crate) async fn remove_filtered_networks( - &self, - vm_id: &str, - networks: &[Networking], - ) -> Result<()> { - if networks - .iter() - .all(|network| netd_teardown(network, &self.config.cvm).is_none()) + /// Releases every host interface netd holds for this VM. + /// + /// Unconditional and non-fatal, which is one decision made twice. The + /// release path must not be gated on a predicate that can change under it: + /// `needs_netd_interface` reads the VM's *current* backend, and a VM whose + /// NIC was a bridge when its TAP was built and is a user-mode NIC now would + /// skip the release for interfaces that exist. And a VM must be able to + /// stop when the daemon holding its interfaces cannot be reached, or a + /// netd outage becomes a fleet that cannot be stopped. + /// + /// Returns whether netd is known to hold nothing for this VM any more. + /// A removal reads that to decide whether it may delete the workdir: the + /// directory is what says to try again, and deleting it over a failed + /// release is what strands an interface with nothing left to reach it. + pub(crate) async fn release_vm_interfaces(&self, vm_id: &str) -> bool { + // Ask for the release, rather than asking whether it can be asked for. + // A probe first would put a second round trip in front of every stop + // and -- worse -- would make a *busy* netd look like an absent one and + // skip the release entirely. The operation itself cannot be misread + // that way: it succeeds, or it says netd is not there, or netd answers + // with a refusal, and only the last of those has a fallback. + match netd::remove_all( + &self.config.netd.socket, + &self.config.cvm.instance_id, + vm_id, + ) + .await { - return Ok(()); - } - let mut first_error = None; - for (nic_index, network) in networks.iter().enumerate().rev() { - let Some(filtered) = netd_teardown(network, &self.config.cvm) else { - continue; - }; - let identity = InterfaceIdentity { - instance_id: self.config.cvm.instance_id.clone(), - vm_id: vm_id.to_string(), - nic_index, - }; - if let Err(error) = netd::request( - &self.config.netd.socket, - &NetdRequest::Remove { identity, filtered }, - ) - .await - { - first_error.get_or_insert(error); + Ok(removed) => { + if removed > 0 { + info!(vm_id, removed, "released netd-managed interfaces"); + } + true + } + Err(error) if netd::is_unreachable(&error) => { + debug!(vm_id, %error, "no netd to release interfaces from"); + false + } + Err(error) => { + warn!( + vm_id, + "failed to release netd-managed interfaces: {error:#}" + ); + false } } - if let Some(error) = first_error { - return Err(error).context("failed to remove netd-managed networking"); - } - Ok(()) } pub(crate) async fn stop_vm_process(&self, id: &str) -> Result<()> { @@ -873,12 +924,11 @@ impl App { pub async fn remove_vm(&self, id: &str) -> Result<()> { { let mut state = self.lock(); - let vm = state.get_mut(id).context("VM not found")?; - if vm.state.removing { + state.get(id).context("VM not found")?; + if !state.start_removing(id) { // Already being removed — idempotent return Ok(()); } - vm.state.removing = true; } // Persist the removing marker so crash recovery can resume @@ -904,13 +954,31 @@ impl App { /// /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup. async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> { + // Every exit from here clears the mark, including the `?`s below and a + // panic in this task, which `tokio::spawn` would otherwise swallow. + let _mark = RemovalMark { + app: self.clone(), + id: id.to_string(), + }; + // Held across the stop, the wait and the release, not just the release. + // `removing` turns launches away, but a launch that passed that check + // before the marker was set is already inside the lock: it has not + // deployed yet, so the wait below sees nothing running and returns at + // once, and the release then deletes the interfaces of the QEMU that + // launch went on to start. Taking the lock first means the launch + // finishes before removal decides anything, and removal then stops what + // it actually started. + let _launch = self.launch_lock(id).await; // Stop the supervisor process (idempotent if already stopped) if let Err(err) = self.stop_vm_process(id).await { debug!("graceful VM stop during removal failed: {err:?}"); } - // Poll until the process is no longer running, then remove it. - // Some VMs take a long time to stop (e.g. 2+ hours), so we wait indefinitely. + // Poll until the process is no longer running, then remove it. The + // stop above is a SIGKILL, so this is however long the kernel takes to + // tear the VM down -- seconds for a large TD, unbounded for one wedged + // in a device reset. Waiting is still right: what follows deletes the + // interfaces and the workdir it is using. let mut poll_count: u64 = 0; loop { match self.supervisor.info(id).await { @@ -942,25 +1010,37 @@ impl App { } } - let runtime_networks = self.work_dir(id)?.runtime_networks(); - if let Err(error) = self.remove_filtered_networks(id, &runtime_networks).await { - warn!(id, %error, "failed to remove filtered networking during VM removal"); - } + let vm_path = self.work_dir(id)?; + // Read before the release, because the release is what makes it stale. + // A VM that never asked netd for an interface -- user mode, a custom + // netdev, or one that never launched -- has nothing for netd to be + // holding, so an absent netd is not a reason to keep its directory. + let held_interfaces = vm_path.runtime_networks().iter().any(needs_netd_interface); + let released = self.release_vm_interfaces(id).await; // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. - let vm_path = self.work_dir(id)?; - if delete_workdir || vm_path.is_removing() { + if !(delete_workdir || vm_path.is_removing()) { if vm_path.path().exists() { - if let Err(err) = fs::remove_dir_all(&vm_path) { - error!("failed to remove VM directory for {id}: {err:?}"); - } + info!( + "VM {id} workdir preserved (orphan cleanup): {}", + vm_path.path().display() + ); } - } else if vm_path.path().exists() { - info!( - "VM {id} workdir preserved (orphan cleanup): {}", - vm_path.path().display() + } else if held_interfaces && !released { + // The `.removing` marker and the directory are what a later boot + // reads to retry this, and `remove_all` is idempotent, so keeping + // them costs one retry and losing them strands every interface + // netd still holds: nothing else on the host can name them. + warn!( + "VM {id} keeps its directory because netd did not release its interfaces; \ + the removal resumes at the next VMM start" ); + return Ok(()); + } else if vm_path.path().exists() { + if let Err(err) = fs::remove_dir_all(&vm_path) { + error!("failed to remove VM directory for {id}: {err:?}"); + } } // Free CID and remove from memory (last step) @@ -980,16 +1060,13 @@ impl App { /// Returns false if a cleanup task is already running for this VM. fn spawn_finish_remove(&self, id: &str) -> bool { { - let mut state = self.lock(); - if let Some(vm) = state.get_mut(id) { - if vm.state.removing { - // Already being cleaned up — skip - return false; - } - vm.state.removing = true; + // Claimed in the set rather than in the entry: an orphaned + // supervisor process has no entry, and that is exactly the case + // where the launch lock is held with nothing turning waiters away. + if !self.lock().start_removing(id) { + // Already being cleaned up — skip + return false; } - // If VM is not in memory (e.g. orphaned supervisor process), no entry to guard - // but we still need to clean up the supervisor process. } let app = self.clone(); let id = id.to_string(); @@ -1325,14 +1402,11 @@ impl App { }); let total = infos.len() as u32; - // One probe for the whole page, and none at all when every VM is - // running and has its own snapshot to report. - let netd_reachable = OnceCell::new(); let vms = paginate(infos, request.page, request.page_size) .map(|vm| { let work_dir = self.work_dir(&vm.config.manifest.id)?; let info = vm.merged_info(vms.get(&vm.config.manifest.id), &work_dir); - let networks = self.effective_networks(&info, &netd_reachable); + let networks = self.effective_networks(&info); Ok(info.to_pb(&self.config.gateway, request.brief, &networks)) }) .collect::>>()?; @@ -1357,9 +1431,9 @@ impl App { pub async fn vm_info(&self, id: &str) -> Result> { let proc_state = self.supervisor.info(id).await?; - // Snapshot under the lock, then release it: describing the VM can - // probe netd, and that is a blocking connect the global state lock has - // no business being held across. + // Snapshot under the lock, then release it: the global state lock is + // held by every other VM's operations, and describing one VM has no + // business keeping it across the work that follows. let info = { let state = self.lock(); let Some(vm_state) = state.get(id) else { @@ -1367,8 +1441,7 @@ impl App { }; vm_state.merged_info(proc_state.as_ref(), &self.work_dir(id)?) }; - let netd_reachable = OnceCell::new(); - let networks = self.effective_networks(&info, &netd_reachable); + let networks = self.effective_networks(&info); Ok(Some(info.to_pb(&self.config.gateway, false, &networks))) } @@ -2064,6 +2137,205 @@ mod tests { use super::mr_config::{mr_config_version, MrConfigVersion}; use super::*; + fn test_app() -> App { + use rocket::figment::providers::Format as _; + let config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + App::new(config, SupervisorClient::new("http://127.0.0.1:0")) + } + + fn test_config(netd_socket: &Path, run_path: &Path) -> Config { + use rocket::figment::providers::Format as _; + let mut config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + config.netd.socket = netd_socket.to_path_buf(); + config.cvm.instance_id = "test-instance".to_string(); + config.run_path = run_path.to_path_buf(); + config + } + + fn app_talking_to(netd_socket: &Path) -> App { + let run_path = netd_socket.parent().unwrap_or(Path::new("/nonexistent")); + App::new( + test_config(netd_socket, run_path), + SupervisorClient::new("http://127.0.0.1:0"), + ) + } + + /// A netd outage must not become a fleet that cannot be stopped. + #[tokio::test] + async fn a_stop_survives_a_netd_that_is_not_there() { + let app = app_talking_to(Path::new("/nonexistent/dstack-netd.sock")); + // Returns rather than propagating: there is no error type here on + // purpose, because there is no caller that should act on one. + app.release_vm_interfaces("vm-1").await; + } + + /// A removal deletes the workdir, and the workdir is the only thing left + /// that says to retry. So the release has to say whether it landed: an + /// answer means netd holds nothing, and anything else -- a refusal, or a + /// netd that is not there to ask -- means it may still. + #[tokio::test] + async fn a_release_says_whether_netd_still_holds_anything() { + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); + let app = app_talking_to(netd.socket()); + assert!(app.release_vm_interfaces("vm-1").await); + + // Refused: netd is up and still holding whatever it had. + let refusing = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); + let app = app_talking_to(refusing.socket()); + assert!(!app.release_vm_interfaces("vm-1").await); + + // Not there to ask. A VM that never asked netd for an interface is + // unaffected -- the removal checks that separately -- but one that did + // must keep its directory so a later start can try again. + let app = app_talking_to(Path::new("/nonexistent/dstack-netd.sock")); + assert!(!app.release_vm_interfaces("vm-1").await); + } + + /// A netd too old for the sweep refuses it, and a refusal is not a reason + /// to fail the stop. What it holds stays until this VM launches again or + /// until an operator names it. The stop still succeeds: a netd outage must + /// not become a fleet that cannot be stopped. + #[tokio::test] + async fn a_netd_that_refuses_the_sweep_does_not_fail_the_stop() { + let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); + let app = app_talking_to(netd.socket()); + app.release_vm_interfaces("vm-1").await; + assert_eq!( + netd.operations(), + vec!["remove_all"], + "asked once, and not asked about afterwards" + ); + } + + /// One request, no question in front of it. A probe on the hot path is a + /// second round trip whose failure mode is silence: a netd too slow to + /// answer it reads as an absent one, and an absent netd's release is + /// skipped. + #[tokio::test] + async fn a_stop_asks_netd_to_sweep_without_a_question_first() { + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); + let app = app_talking_to(netd.socket()); + app.release_vm_interfaces("vm-1").await; + + assert_eq!(netd.operations(), vec!["remove_all"]); + let sweep = &netd.seen()[0]; + assert_eq!(sweep["vm_id"], "vm-1"); + assert_eq!(sweep["instance_id"], "test-instance"); + // A sweep names no NIC: reaching the indices the caller can no longer + // name is the entire point. + assert!(sweep.get("nic_index").is_none()); + } + + /// The orphan cleanup runs for an ID that never loaded, so a guard living + /// in the VM entry is not there when it holds the launch lock across the + /// whole teardown. Without the mark, `StartVm` on that ID waits the + /// teardown out with no error and no log. + #[tokio::test] + async fn a_removal_with_no_vm_entry_still_turns_operations_away() { + let app = test_app(); + assert!(app.refuse_if_removing("orphan").is_ok()); + assert!(app.lock().start_removing("orphan")); + assert!( + app.refuse_if_removing("orphan").is_err(), + "an orphan being removed is still a VM being removed" + ); + // And the same removal cannot be started twice. + assert!(!app.lock().start_removing("orphan")); + } + + /// `finish_remove_vm` returns early on more than its happy path. A mark it + /// left behind is not a stale flag: every later operation on that VM, + /// including the removal that would retry, answers "being removed". + #[tokio::test] + async fn a_removal_that_gives_up_early_does_not_leave_the_vm_marked() { + let app = test_app(); + { + let _mark = RemovalMark { + app: app.clone(), + id: "vm-1".to_string(), + }; + assert!(app.lock().start_removing("vm-1")); + assert!(app.refuse_if_removing("vm-1").is_err()); + } + assert!( + app.refuse_if_removing("vm-1").is_ok(), + "the mark is cleared however the removal ends" + ); + } + + /// A restart decided before a stop must not outlive it. The restart task + /// reads the started flag off disk and only then queues a launch, which + /// waits for the lock the stop is holding; without a re-read under that + /// lock the launch resurrects a VM the operator was told was stopped. + #[tokio::test] + async fn an_automatic_restart_does_not_outlive_the_stop_it_raced() { + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(Path::new("/nonexistent/netd.sock"), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + let work_dir = app.work_dir("vm-1").unwrap(); + std::fs::create_dir_all(work_dir.path()).unwrap(); + + work_dir.set_started(false).unwrap(); + app.start_vm_with_restart_policy("vm-1", false) + .await + .expect("an automatic restart of a stopped VM does nothing"); + + // The flag is the whole difference: with it set, the same call goes on + // to do the work, and fails here for want of a VM to launch. + work_dir.set_started(true).unwrap(); + assert!(app + .start_vm_with_restart_policy("vm-1", false) + .await + .is_err()); + + // An explicit start sets the flag itself and has nothing to re-read, + // so it is never turned away by one. + work_dir.set_started(false).unwrap(); + assert!(app.start_vm("vm-1").await.is_err()); + } + + /// The window a launch spends between reading "not running" and actually + /// starting QEMU is long -- a GPU reset, a netd conversation -- and the + /// sweep inside it deletes interfaces by deriving their names. Two entrants + /// in that window meant the loser deleting the winner's live TAPs. + #[tokio::test] + async fn one_vm_launches_at_a_time_and_the_lock_map_stays_small() { + let app = test_app(); + let held = app.launch_lock("vm-1").await; + + // A different VM is never blocked by it: a slow start must not stall + // every other launch on the node. + let other = tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-2")).await; + assert!(other.is_ok(), "an unrelated VM must not wait"); + + // The same VM is. + let same = tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-1")).await; + assert!(same.is_err(), "a second entrant must wait for the first"); + + drop(held); + drop(other); + tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-1")) + .await + .expect("the lock is released"); + + // Nothing is in flight now, so the map holds nothing either. + assert!(app.launch_locks.lock().unwrap().len() <= 1); + let _ = app.launch_lock("vm-3").await; + assert!(app.launch_locks.lock().unwrap().len() <= 2); + } + #[test] fn accepts_server_generated_ids() { validate_vm_id(&uuid::Uuid::new_v4().to_string()).unwrap(); @@ -3158,6 +3430,14 @@ impl VmState { pub(crate) struct AppState { cid_pool: IdPool, vms: HashMap, + /// The VMs a removal is currently working on. + /// + /// Separate from `VmState::removing` because the set has to outlive the + /// entry. Orphan cleanup runs for IDs that never loaded into `vms`, and + /// `finish_remove_vm` holds the launch lock across the whole teardown, so + /// a guard that lives in the entry cannot turn away the operation that + /// would otherwise wait that teardown out. + removing: HashSet, } impl AppState { @@ -3180,6 +3460,38 @@ impl AppState { pub fn iter_vms(&self) -> impl Iterator { self.vms.values() } + + /// Claims `id` for a removal. False when one already has it. + fn start_removing(&mut self, id: &str) -> bool { + if let Some(vm) = self.vms.get_mut(id) { + vm.state.removing = true; + } + self.removing.insert(id.to_string()) + } + + fn is_removing(&self, id: &str) -> bool { + self.removing.contains(id) + } +} + +/// Clears the in-flight removal mark however the removal ends. +/// +/// `finish_remove_vm` returns early on more than its happy path, and a mark +/// left behind is not a stale flag: every operation on that VM answers "being +/// removed" from then on, including the removal that would retry. +struct RemovalMark { + app: App, + id: String, +} + +impl Drop for RemovalMark { + fn drop(&mut self) { + let mut state = self.app.lock(); + state.removing.remove(&self.id); + if let Some(vm) = state.vms.get_mut(&self.id) { + vm.state.removing = false; + } + } } /// Reject VM ids that would escape `run_path` once joined into a filesystem diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index 4017e89ff..b946e3ca1 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -9,10 +9,9 @@ use std::path::Path; use anyhow::{bail, Result}; use sha2::{Digest, Sha256}; -use super::Manifest; +use super::{Manifest, PortMapping}; use crate::config::{ - CvmConfig, NetdInterface, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, - MAX_NET_QUEUES, + CvmConfig, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, MAX_NET_QUEUES, }; /// Node configuration merged with what one NIC pins. @@ -39,7 +38,6 @@ pub(crate) fn resolve_networking( // Runtime state, never inherited from configuration or from a previous // launch. Interface preparation sets both for the NICs it builds, and a // node configuration that names either is rejected at startup. - resolved.netd_interface = crate::config::NetdInterface::None; resolved.device.clear(); if !networking.bridge.is_empty() { resolved.nic.bridge = networking.bridge.clone(); @@ -85,17 +83,22 @@ pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec bool { - match networking.nic.mode { - NetworkingMode::Macvtap => true, - NetworkingMode::Bridge => { - cfg.network_filter.mode == NetworkFilterMode::Libvirt || networking.queue_pairs() > 1 - } - NetworkingMode::User | NetworkingMode::Custom => false, - } +/// Every macvtap and every bridge NIC. The alternative was a set of conditions +/// -- libvirt filtering, multiqueue -- under which netd was consulted and +/// outside of which the VMM built the interface some other way. Each of those +/// paths had to answer the same questions again and answer them differently: +/// which netdev QEMU gets, whether vhost is really on, and, once port mappings +/// grew a NIC, where a bridge NIC's host ports go. A host interface has one +/// owner now, and `port_map` on a bridge reaches netd on every node rather than +/// only on the ones that happened to filter or to have scaled their queues. +/// +/// The cost is stated plainly: bridge and macvtap need a netd on the host. User +/// mode and a caller-supplied netdev still need nothing. +pub(crate) fn needs_netd_interface(networking: &Networking) -> bool { + matches!( + networking.nic.mode, + NetworkingMode::Macvtap | NetworkingMode::Bridge + ) } /// Whether this NIC's host interface carries a libvirt nwfilter binding. @@ -105,146 +108,16 @@ pub(crate) fn filters_bridge_traffic(networking: &Networking, cfg: &CvmConfig) - && cfg.network_filter.mode == NetworkFilterMode::Libvirt } -/// Whether netd built this NIC's host interface, and if so whether it carries -/// an nwfilter binding. -/// -/// Interface preparation records this, because it is not derivable afterwards: -/// an operator can change `network_filter.mode` or `max_net_queues` while a VM -/// runs, and teardown has to undo what was built rather than what would be -/// built now. -pub(crate) fn netd_teardown(networking: &Networking, cfg: &CvmConfig) -> Option { - match networking.netd_interface { - NetdInterface::Filtered => Some(true), - NetdInterface::Unfiltered => Some(false), - // Either nothing was built, or this entry was persisted before - // preparation recorded the fact. Fall back to the derivation such an - // entry was created by; a Remove for an interface that does not exist - // is a no-op. - NetdInterface::None if needs_netd_interface(networking, cfg) => { - Some(filters_bridge_traffic(networking, cfg)) - } - NetdInterface::None => None, - } -} - -/// Drops a NIC back to one queue pair when multiqueue would need a netd -/// interface this node cannot provide. -/// -/// Queue pairs are a default now, not something the operator asked for, so a -/// node that has never deployed netd must keep launching bridge VMs. An -/// explicit per-VM request is left alone: the caller asked for it, and failing -/// at prepare tells them why far better than silently halving their throughput. -/// Returns how many NICs it dropped, so a launch can say so and a status -/// query, which runs the same calculation to describe a stopped VM, stays -/// silent. -pub(crate) fn clamp_queues_without_netd( - requested: &[NicNetworking], - resolved: &mut [Networking], - cfg: &CvmConfig, - netd_available: bool, -) -> usize { - if netd_available { - return 0; - } - let mut clamped = 0; - for (networking, asked) in resolved.iter_mut().zip(requested) { - // Macvtap has nothing to fall back to: netd is the only thing that can - // create the device, so clamping one would describe a VM that cannot - // start either way. - if networking.nic.mode != NetworkingMode::Bridge - || asked.queues.is_some() - || !needs_netd_interface(networking, cfg) - // Filtering needs netd whatever the queue count, so dropping this - // NIC to one queue pair would not make it launchable. It would only - // describe it as something no launch can produce, and warn about a - // fallback that is not happening. - || filters_bridge_traffic(networking, cfg) - { - continue; - } - networking.nic.queues = Some(1); - clamped += 1; - } - clamped -} - -/// Locations distributions install `qemu-bridge-helper` in. The helper is -/// setuid root and attaches an unprivileged TAP to a whitelisted bridge, which -/// is how bridge mode avoids giving the VMM `CAP_NET_ADMIN`. -const BRIDGE_HELPER_CANDIDATES: [&str; 3] = [ - "/usr/lib/qemu/qemu-bridge-helper", - "/usr/libexec/qemu-bridge-helper", - "/usr/local/libexec/qemu-bridge-helper", -]; - -/// Absolute path of `qemu-bridge-helper`, which QEMU's `tap` netdev, unlike its -/// `bridge` netdev, has no compiled-in default for. -/// -/// A configured path is passed through unchecked: the operator is naming a -/// binary for QEMU to exec, and QEMU need not see this filesystem. -pub(crate) fn find_bridge_helper<'a>( - configured: &'a str, - candidates: &[&'a str], -) -> Option<&'a str> { - let configured = configured.trim(); - if !configured.is_empty() { - return Some(configured); - } - candidates - .iter() - .copied() - .find(|candidate| Path::new(candidate).exists()) -} - -pub(crate) fn bridge_helper(cfg: &CvmConfig) -> Option<&str> { - find_bridge_helper(&cfg.qemu_bridge_helper, &BRIDGE_HELPER_CANDIDATES) -} - -/// Whether this NIC will actually run on the vhost-net data plane. -/// -/// A bridge NIC that neither needs a netd interface nor can find -/// `qemu-bridge-helper` falls back to QEMU's `bridge` netdev, which has no -/// vhost support. Both the QEMU arguments and the reported status read this, -/// so a VM is never described as using a data plane it did not get. -pub(crate) fn effective_vhost(networking: &Networking, cfg: &CvmConfig) -> bool { - if !networking.vhost_enabled() { - return false; - } - networking.nic.mode != NetworkingMode::Bridge - || needs_netd_interface(networking, cfg) - || bridge_helper(cfg).is_some() -} - -/// Makes the effective data plane concrete on a launch-time NIC list, and -/// returns how many interfaces asked for vhost and did not get it. +/// Makes the data plane concrete on a launch-time NIC list. /// /// `vhost` on a freshly resolved entry is still a *request*: `None` means -/// inherit, and a bridge NIC that cannot reach `qemu-bridge-helper` runs on the -/// non-vhost netdev whatever it asked for. Settling it once, here, is what lets -/// the QEMU arguments and the reported status read the same value -- and keeps -/// them reading it after the operator moves the helper out from under a VM that -/// is already running. -pub(crate) fn settle_vhost(networks: &mut [Networking], cfg: &CvmConfig) -> usize { - let mut denied = 0; +/// inherit from the node, which can change under a VM that is already running. +/// Settling it once, here, is what lets the QEMU arguments and the reported +/// status read the same value for the life of a boot. +pub(crate) fn settle_vhost(networks: &mut [Networking]) { for networking in networks.iter_mut() { - let effective = effective_vhost(networking, cfg); - if networking.vhost_enabled() && !effective { - denied += 1; - } - networking.nic.vhost = Some(effective); + networking.nic.vhost = Some(networking.vhost_enabled()); } - denied -} - -/// Whether netd is reachable. A netd that died leaves its socket behind, so -/// existence alone would report a node as capable and fail every launch. -/// -/// A connect and nothing more, deliberately: netd serves connections serially, -/// so anything that waits for an answer reads a *busy* netd as a missing one -/// and silently drops the VM to a single queue pair. Accepting the connection -/// is the one signal that does not depend on what netd is doing right now. -pub(crate) fn netd_available(socket: &Path) -> bool { - std::os::unix::net::UnixStream::connect(socket).is_ok() } pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { @@ -329,6 +202,57 @@ pub(crate) fn warn_if_vhost_net_missing(networks: &[Networking]) { } } +/// Which NIC an unpinned port mapping's traffic enters through. +/// +/// The first user-mode NIC, which is where QEMU's `hostfwd=` entries have +/// always gone. There is no second choice: nothing else on this host publishes +/// a port, so a VM without one has nowhere to put a mapping and the launch +/// says so. +pub(crate) fn default_ingress_nic(networks: &[Networking]) -> Option { + networks + .iter() + .position(|network| network.nic.mode == NetworkingMode::User) +} + +/// Whether a NIC of this mode has a mechanism to publish a host port at all. +/// +/// QEMU's `hostfwd=`, and nothing else. A bridge TAP is built by netd, and the +/// netd in this repository does not forward host ports; macvtap bypasses the +/// host bridge; a custom netdev is a string the VMM does not interpret. +pub(crate) fn mode_carries_ingress(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::User) +} + +/// Which NIC a port mapping's traffic enters through. +/// +/// One mapping resolves to at most one NIC, and only a user-mode NIC has a +/// mechanism to carry it, so a pin to any other kind resolves to nothing +/// rather than to a NIC with no path into the guest. +/// +/// `None` is a mapping with nowhere to go: a VM with no user-mode NIC, or one +/// whose NICs changed under a mapping that named one. The launch warns about +/// each of those rather than dropping it in silence. +pub(crate) fn ingress_nic(mapping: &PortMapping, networks: &[Networking]) -> Option { + mapping + .nic_index + .or_else(|| default_ingress_nic(networks)) + .filter(|index| { + networks + .get(*index) + .is_some_and(|network| mode_carries_ingress(network.nic.mode)) + }) +} + +/// Names the mappings that resolve to no NIC, for a launch to warn about. +pub(crate) fn stranded_ingress<'a>( + port_map: &'a [PortMapping], + networks: &'a [Networking], +) -> impl Iterator { + port_map + .iter() + .filter(|mapping| ingress_nic(mapping, networks).is_none()) +} + /// Derives a deterministic, locally administered unicast MAC address. /// /// Index zero preserves the legacy single-NIC derivation. Later interfaces @@ -356,10 +280,11 @@ pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) #[cfg(test)] mod tests { use super::{ - clamp_queues_without_netd, effective_vhost, mac_address_for_vm_index, needs_netd_interface, - netd_teardown, resolve_networking, resolved_networks, settle_vhost, - validate_resolved_networks, + default_ingress_nic, ingress_nic, mac_address_for_vm_index, needs_netd_interface, + resolved_networks, settle_vhost, stranded_ingress, validate_resolved_networks, }; + use crate::app::PortMapping; + use crate::config::Protocol; use crate::config::{Networking, NetworkingMode, NicNetworking}; fn macvtap_network() -> NicNetworking { @@ -450,7 +375,6 @@ mod tests { let resolved = resolved_networks(&manifest_with(16, vec![]), &cvm); assert!(!resolved[0].vhost_enabled()); assert_eq!(resolved[0].queue_pairs(), 1); - assert!(!needs_netd_interface(&resolved[0], &cvm)); } } @@ -461,7 +385,6 @@ mod tests { let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); assert!(!resolved[0].vhost_enabled()); assert_eq!(resolved[0].queue_pairs(), 1); - assert!(!needs_netd_interface(&resolved[0], &cvm)); // Per-VM opt-out does the same thing. let cvm = node_config(NetworkingMode::Bridge); @@ -492,31 +415,37 @@ mod tests { assert_eq!(resolved[0].queue_pairs(), 16); } + /// One owner for a host interface, with no condition attached. What used + /// to decide this -- libvirt filtering, a scaled queue count -- decided it + /// per node, so the same VM definition got a netd-built TAP on one host and + /// a `qemu-bridge-helper` TAP on the next. #[test] - fn status_never_claims_a_data_plane_the_nic_did_not_get() { + fn every_bridge_and_macvtap_nic_is_netds_to_build() { let mut cvm = node_config(NetworkingMode::Bridge); - // No helper on this filesystem and no netd interface needed, so the - // NIC falls back to QEMU's `bridge` netdev, which has no vhost. - cvm.qemu_bridge_helper = String::new(); let mut single = cvm.networking.nic.clone(); single.queues = Some(1); + let resolved = resolved_networks(&manifest_with(8, vec![single.clone()]), &cvm); + assert!(needs_netd_interface(&resolved[0])); + + // Unfiltered, single queue, no vhost -- the shape that used to need no + // netd at all -- is netd's too. + cvm.networking.nic.vhost = Some(false); + single.vhost = Some(false); let resolved = resolved_networks(&manifest_with(8, vec![single]), &cvm); - assert!(resolved[0].vhost_enabled()); - let fell_back = !effective_vhost(&resolved[0], &cvm); - assert_eq!(fell_back, super::bridge_helper(&cvm).is_none()); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + assert!(needs_netd_interface(&resolved[0])); - // A configured helper is taken at its word, so vhost is real. - cvm.qemu_bridge_helper = "/opt/qemu-bridge-helper".into(); + let cvm = node_config(NetworkingMode::Macvtap); let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); - assert!(effective_vhost(&resolved[0], &cvm)); + assert!(needs_netd_interface(&resolved[0])); - // Multiqueue goes through netd, which needs no helper at all. - let mut mq = cvm.networking.nic.clone(); - mq.queues = Some(4); - cvm.qemu_bridge_helper = String::new(); - let resolved = resolved_networks(&manifest_with(8, vec![mq]), &cvm); - assert!(needs_netd_interface(&resolved[0], &cvm)); - assert!(effective_vhost(&resolved[0], &cvm)); + // The two backends the VMM builds itself still need nothing. + for mode in [NetworkingMode::User, NetworkingMode::Custom] { + let cvm = node_config(mode); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(!needs_netd_interface(&resolved[0])); + } } #[test] @@ -536,35 +465,6 @@ mod tests { assert_eq!(resolved[0].queue_pairs(), 1); } - #[test] - fn without_netd_a_defaulted_bridge_drops_to_one_queue_but_a_request_does_not() { - let cvm = node_config(NetworkingMode::Bridge); - - // The default is ours to lower: a node that never deployed netd must - // keep launching bridge VMs. - let requested = vec![cvm.networking.nic.clone()]; - let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); - assert_eq!(resolved[0].queue_pairs(), 8); - clamp_queues_without_netd(&requested, &mut resolved, &cvm, false); - assert_eq!(resolved[0].queue_pairs(), 1); - assert!(!needs_netd_interface(&resolved[0], &cvm)); - - // An explicit request is left alone, so prepare fails where the caller - // can see why instead of silently halving their throughput. - let mut asked = cvm.networking.nic.clone(); - asked.queues = Some(4); - let requested = vec![asked.clone()]; - let mut resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); - clamp_queues_without_netd(&requested, &mut resolved, &cvm, false); - assert_eq!(resolved[0].queue_pairs(), 4); - - // With netd present nothing is touched. - let requested = vec![cvm.networking.nic.clone()]; - let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); - clamp_queues_without_netd(&requested, &mut resolved, &cvm, true); - assert_eq!(resolved[0].queue_pairs(), 8); - } - #[test] fn validation_never_depends_on_this_process_reaching_vhost_net() { // QEMU may run under different credentials, so a NIC that asks for @@ -593,133 +493,113 @@ mod tests { #[test] fn settling_vhost_records_what_the_launch_decided() { let mut cvm = node_config(NetworkingMode::Bridge); - cvm.qemu_bridge_helper = String::new(); - let mut single = cvm.networking.nic.clone(); - single.queues = Some(1); - let manifest = manifest_with(8, vec![single]); - - // Whether this host has a helper is not the test's business; that it - // gets written down, once, is. - let helper_missing = super::bridge_helper(&cvm).is_none(); + let manifest = manifest_with(8, vec![]); let mut networks = resolved_networks(&manifest, &cvm); - assert!(networks[0].vhost_enabled(), "the request starts out on"); - assert_eq!( - settle_vhost(&mut networks, &cvm), - usize::from(helper_missing) - ); - assert_eq!(networks[0].nic.vhost, Some(!helper_missing)); - // Settling an already-settled list reports nothing new, so a relaunch - // does not warn about a fallback that already happened. - assert_eq!(settle_vhost(&mut networks, &cvm), 0); + assert_eq!(networks[0].nic.vhost, Some(true)); + settle_vhost(&mut networks); + assert_eq!(networks[0].nic.vhost, Some(true)); - // A configured helper is taken at its word, so the same NIC settles on. - cvm.qemu_bridge_helper = "/opt/qemu-bridge-helper".into(); - let mut with_helper = resolved_networks(&manifest, &cvm); - assert_eq!(settle_vhost(&mut with_helper, &cvm), 0); - assert_eq!(with_helper[0].nic.vhost, Some(true)); + // The node turns vhost off under a VM that is already running. The + // entry the launch settled keeps its answer; the next boot gets the + // new one. + cvm.networking.nic.vhost = Some(false); + assert_eq!(networks[0].nic.vhost, Some(true)); + let mut next_boot = resolved_networks(&manifest, &cvm); + settle_vhost(&mut next_boot); + assert_eq!(next_boot[0].nic.vhost, Some(false)); - // The entry the first launch settled keeps its answer: nothing about a - // running VM is recomputed from the configuration as it stands now. - assert_eq!(networks[0].nic.vhost, Some(!helper_missing)); + // An inherited `None` becomes a decision rather than staying a request. + let mut unset = resolved_networks(&manifest, &cvm); + unset[0].nic.vhost = None; + settle_vhost(&mut unset); + assert_eq!(unset[0].nic.vhost, Some(false)); } - /// Dropping to a single queue pair is only worth doing when it makes the - /// NIC launchable. A filtered bridge needs netd whatever its queue count, - /// so clamping it would report a shape no launch can produce. #[test] - fn a_filtered_bridge_is_not_clamped_because_it_cannot_help() { - use crate::config::NetworkFilterMode; - - let mut cvm = node_config(NetworkingMode::Bridge); - cvm.network_filter.mode = NetworkFilterMode::Libvirt; - let requested = vec![cvm.networking.nic.clone()]; - let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); - assert_eq!(resolved[0].queue_pairs(), 8); + fn primary_mac_keeps_legacy_derivation_and_later_nics_are_distinct() { assert_eq!( - clamp_queues_without_netd(&requested, &mut resolved, &cvm, false), - 0 + mac_address_for_vm_index("vm-123", &[], 0), + "96:b1:d8:b9:08:e6" ); - assert_eq!(resolved[0].queue_pairs(), 8); - - // Unfiltered, the same NIC does drop, because then it can launch. - let cvm = node_config(NetworkingMode::Bridge); - let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); assert_eq!( - clamp_queues_without_netd(&requested, &mut resolved, &cvm, false), - 1 + mac_address_for_vm_index("vm-123", &[], 1), + "c6:74:2c:65:14:b9" ); - assert_eq!(resolved[0].queue_pairs(), 1); } - /// Teardown has to undo what was built. Node configuration is mutable and - /// a VM outlives an edit to it, so re-deriving "did netd build this?" at - /// removal time orphans TAPs and leaks nwfilter bindings whose ebtables - /// rules the next VM at the same deterministic interface name inherits. - #[test] - fn teardown_follows_what_was_built_not_what_configuration_now_says() { - use crate::config::{NetdInterface, NetworkFilterMode}; + fn nic(mode: NetworkingMode) -> Networking { + Networking { + nic: NicNetworking { + mode, + ..NicNetworking::default() + }, + ..Networking::default() + } + } + + fn mapping(host_port: u16, nic_index: Option) -> PortMapping { + PortMapping { + address: "0.0.0.0".parse().unwrap(), + protocol: Protocol::Tcp, + from: host_port, + to: host_port, + nic_index, + } + } - let filtering = { - let mut cvm = node_config(NetworkingMode::Bridge); - cvm.network_filter.mode = NetworkFilterMode::Libvirt; - cvm - }; - let unfiltered = node_config(NetworkingMode::Bridge); - - let mut built_filtered = filtering.networking.clone(); - built_filtered.nic.queues = Some(1); - built_filtered.netd_interface = NetdInterface::Filtered; - // The operator turns filtering off while the VM runs. The binding is - // still there and still has to be deleted. - assert_eq!(netd_teardown(&built_filtered, &unfiltered), Some(true)); - - let mut built_unfiltered = unfiltered.networking.clone(); - built_unfiltered.nic.queues = Some(4); - built_unfiltered.netd_interface = NetdInterface::Unfiltered; - // The operator turns filtering on. There is no binding to delete, and - // asking libvirt for one would fail the removal. - assert_eq!(netd_teardown(&built_unfiltered, &filtering), Some(false)); - - // A NIC netd never touched stays untouched, whatever the node now says. - let mut untouched = unfiltered.networking.clone(); - untouched.nic.queues = Some(1); - assert_eq!(netd_teardown(&untouched, &unfiltered), None); - - // An entry persisted before preparation recorded the fact still gets - // torn down by the rule that created it. - let mut legacy = filtering.networking.clone(); - legacy.nic.queues = Some(1); - assert_eq!(legacy.netd_interface, NetdInterface::None); - assert_eq!(netd_teardown(&legacy, &filtering), Some(true)); - } - - /// Resolution produces launch input, never a claim about what exists. #[test] - fn resolution_never_carries_a_stale_interface_record() { - use crate::config::NetdInterface; + fn an_unpinned_mapping_still_lands_where_hostfwd_always_put_it() { + // Existing VMs must not move. QEMU's `hostfwd=` has always gone to the + // first user-mode NIC, so that stays the answer wherever there is one. + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; + assert_eq!(default_ingress_nic(&networks), Some(1)); + assert_eq!(ingress_nic(&mapping(443, None), &networks), Some(1)); - let cvm = node_config(NetworkingMode::Bridge); - // Single queue and no filtering, so nothing but a stale record could - // make teardown believe netd built something. - let mut previous = cvm.networking.clone(); - previous.nic.queues = Some(1); - previous.netd_interface = NetdInterface::Filtered; - assert_eq!(netd_teardown(&previous, &cvm), Some(true)); + // With no user-mode NIC there is nowhere at all. netd builds a bridge + // TAP but does not forward host ports, and macvtap and custom have no + // path either. + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::Bridge)]; + assert_eq!(default_ingress_nic(&networks), None); - let resolved = resolve_networking(&previous.nic, &cvm, 4); - assert_eq!(resolved.netd_interface, NetdInterface::None); - assert_eq!(netd_teardown(&resolved, &cvm), None); + let networks = [nic(NetworkingMode::Macvtap), nic(NetworkingMode::Custom)]; + assert_eq!(default_ingress_nic(&networks), None); + assert_eq!(ingress_nic(&mapping(443, None), &networks), None); } #[test] - fn primary_mac_keeps_legacy_derivation_and_later_nics_are_distinct() { - assert_eq!( - mac_address_for_vm_index("vm-123", &[], 0), - "96:b1:d8:b9:08:e6" - ); - assert_eq!( - mac_address_for_vm_index("vm-123", &[], 1), - "c6:74:2c:65:14:b9" - ); + fn a_pinned_mapping_goes_where_it_says() { + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; + assert_eq!(ingress_nic(&mapping(443, Some(1)), &networks), Some(1)); + // Out of range resolves to nothing rather than to something arbitrary. + // Deployment refuses it outright; a manifest that lost a NIC lands here. + assert_eq!(ingress_nic(&mapping(443, Some(7)), &networks), None); + } + + /// A pin has to be checked against the backend, not just the count. + /// Naming a macvtap or custom NIC used to resolve to that index and then + /// fall out of every branch that could act on it: no `hostfwd=`, no netd + /// request, and no warning either. + #[test] + fn a_pin_to_a_backend_with_no_ingress_resolves_to_nothing() { + let networks = [ + nic(NetworkingMode::Macvtap), + nic(NetworkingMode::Custom), + nic(NetworkingMode::Bridge), + nic(NetworkingMode::User), + ]; + assert_eq!(ingress_nic(&mapping(443, Some(0)), &networks), None); + assert_eq!(ingress_nic(&mapping(443, Some(1)), &networks), None); + // A bridge TAP is netd's, and netd does not forward host ports. + assert_eq!(ingress_nic(&mapping(443, Some(2)), &networks), None); + assert_eq!(ingress_nic(&mapping(443, Some(3)), &networks), Some(3)); + + // And an unpinned mapping on a VM with nowhere to put it is named, + // rather than counted as delivered. + let networks = [nic(NetworkingMode::Macvtap)]; + let port_map = [mapping(443, None), mapping(8080, Some(0))]; + let stranded: Vec<_> = stranded_ingress(&port_map, &networks) + .map(|mapping| mapping.from) + .collect(); + assert_eq!(stranded, vec![443, 8080]); } } diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 94a6348fe..fe420a484 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -10,7 +10,7 @@ use super::{ image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, network::{ - bridge_helper, mac_address_for_vm_index, needs_netd_interface, validate_resolved_networks, + ingress_nic, mac_address_for_vm_index, validate_resolved_networks, warn_if_vhost_net_missing, }, pci_numa_node, round_up, GpuConfig, VmWorkDir, @@ -631,11 +631,6 @@ impl QemuCommandBuilder<'_> { fn configure_networking(&self, command: &mut Command) -> Result<()> { let macvtap_fds = macvtap_fd_layout(&self.prepared.networks); - let hostfwd_index = self - .prepared - .networks - .iter() - .position(|networking| networking.nic.mode == NetworkingMode::User); for (index, networking) in self.prepared.networks.iter().enumerate() { let net_id = format!("net{index}"); let mac = mac_address_for_vm_index( @@ -663,16 +658,21 @@ impl QemuCommandBuilder<'_> { networking.dhcp_start, if networking.restrict { "yes" } else { "no" } ); - if hostfwd_index == Some(index) { - for mapping in &self.vm.manifest.port_map { - netdev.push_str(&format!( - ",hostfwd={}:{}:{}-:{}", - mapping.protocol.as_str(), - mapping.address, - mapping.from, - mapping.to - )); + // Only the mappings that resolve to this NIC. A mapping + // lands on exactly one, and that NIC's backend decides the + // mechanism, so a bridge NIC's ports go to netd instead of + // being claimed here as well. + for mapping in &self.vm.manifest.port_map { + if ingress_nic(mapping, &self.prepared.networks) != Some(index) { + continue; } + netdev.push_str(&format!( + ",hostfwd={}:{}:{}-:{}", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.to + )); } netdev } @@ -681,43 +681,25 @@ impl QemuCommandBuilder<'_> { "bridge networking: mac={mac} bridge={} vhost={vhost} queues={queues}", networking.nic.bridge ); - if needs_netd_interface(networking, self.cfg) { - // netd owns this TAP: libvirt filtering binds an - // nwfilter to it, and multiqueue needs the persistent - // IFF_MULTI_QUEUE device the bridge helper cannot make. - let tap = tap_name(&InterfaceIdentity { - instance_id: self.cfg.instance_id.clone(), - vm_id: self.vm.manifest.id.clone(), - nic_index: index, - }); - let mut netdev = format!( - "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost={}", - on_off(vhost) - ); - if queues > 1 { - netdev.push_str(&format!(",queues={queues}")); - } - netdev - } else if let Some(helper) = vhost.then(|| bridge_helper(self.cfg)).flatten() { - // QEMU's `bridge` netdev has no vhost support, but the - // same setuid helper works behind a `tap` netdev, so - // the VMM still needs no network privileges. - format!( - "tap,id={net_id},br={},helper={helper},vhost=on", - networking.nic.bridge - ) - } else if vhost { - // vhost is a node-wide setting, so a node whose helper - // sits somewhere unusual must keep booting VMs rather - // than lose every bridge NIC to a path lookup. - tracing::warn!( - "{net_id}: no qemu-bridge-helper found, falling back to the \ - non-vhost bridge netdev. set cvm.qemu_bridge_helper to enable vhost" - ); - format!("bridge,id={net_id},br={}", networking.nic.bridge) - } else { - format!("bridge,id={net_id},br={}", networking.nic.bridge) + // netd owns the TAP. It is the one component here with + // CAP_NET_ADMIN, so it is the only one that can bind an + // nwfilter or create a persistent IFF_MULTI_QUEUE device -- + // and having it own every bridge TAP is what keeps a VM's + // networking from depending on which of those a node + // happens to use. + let tap = tap_name(&InterfaceIdentity { + instance_id: self.cfg.instance_id.clone(), + vm_id: self.vm.manifest.id.clone(), + nic_index: index, + }); + let mut netdev = format!( + "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost={}", + on_off(vhost) + ); + if queues > 1 { + netdev.push_str(&format!(",queues={queues}")); } + netdev } NetworkingMode::Custom => { if !networking.netdev.contains(&format!("id={net_id}")) { @@ -1192,6 +1174,7 @@ mod tests { protocol: Protocol::Tcp, from: 18080, to: 8080, + nic_index: None, }], created_at_ms: 0, hugepages: false, @@ -1290,16 +1273,23 @@ mod tests { } #[test] - fn bridge_vhost_uses_the_bridge_helper_behind_a_tap_netdev() { - // QEMU's `bridge` netdev has no vhost support at all, so enabling the - // kernel data plane has to switch netdev types while keeping the same - // unprivileged setuid helper. + fn every_bridge_nic_gets_the_netd_tap() { + // Not only the filtered or multiqueue ones. A bridge NIC's host + // interface has one owner, so the netdev QEMU is handed does not + // change with the node's filter mode or its queue count. let (mut config, ..) = test_launch_fixture(); - config.cvm.qemu_bridge_helper = "/usr/lib/qemu/qemu-bridge-helper".into(); - let args = net_args(&config, vec![bridge_network(&config)]); - assert!(args.contains( - &"tap,id=net0,br=br0,helper=/usr/lib/qemu/qemu-bridge-helper,vhost=on".to_string() - )); + config.cvm.instance_id = "vmm-a".into(); + let mut networking = bridge_network(&config); + networking.nic.queues = Some(1); + let args = net_args(&config, vec![networking]); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=on" + ))); // A single queue pair must keep the historical device line byte for byte. assert!(args.iter().any( |arg| arg.starts_with("virtio-net-pci,netdev=net0,mac=") && !arg.contains("mq=on") @@ -1307,26 +1297,20 @@ mod tests { } #[test] - fn a_missing_bridge_helper_is_reported_rather_than_guessed() { - // Configured paths are trusted verbatim: QEMU execs them, and it need - // not share this filesystem. - assert_eq!( - crate::app::network::find_bridge_helper(" /opt/qemu-bridge-helper ", &[]), - Some("/opt/qemu-bridge-helper") - ); - assert_eq!( - crate::app::network::find_bridge_helper("", &["/nonexistent/a", "/nonexistent/b"]), - None - ); - } - - #[test] - fn disabling_vhost_restores_the_legacy_bridge_netdev() { - let (config, ..) = test_launch_fixture(); + fn disabling_vhost_keeps_the_netd_tap_and_turns_the_data_plane_off() { + let (mut config, ..) = test_launch_fixture(); + config.cvm.instance_id = "vmm-a".into(); let mut networking = bridge_network(&config); networking.nic.vhost = Some(false); let args = net_args(&config, vec![networking]); - assert!(args.contains(&"bridge,id=net0,br=br0".to_string())); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=off" + ))); } #[test] @@ -1490,19 +1474,6 @@ mod tests { network.nic.bridge = "br0".into(); network.nic.vhost = Some(false); } - let process = QemuCommandBuilder { - vm: &vm, - cfg: &config.cvm, - gpus: &GpuConfig::default(), - prepared: &prepared, - } - .build() - .unwrap(); - assert!(process - .args - .iter() - .any(|arg| arg == "bridge,id=net0,br=br0")); - config.cvm.instance_id = "vmm-a".into(); config.cvm.network_filter.mode = NetworkFilterMode::Libvirt; let process = QemuCommandBuilder { diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 19d72118a..48af6f864 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -33,15 +33,6 @@ pub(crate) struct VmInfo { pub runtime_networks: Vec, } -fn networking_mode_name(mode: NetworkingMode) -> &'static str { - match mode { - NetworkingMode::Bridge => "bridge", - NetworkingMode::User => "user", - NetworkingMode::Custom => "custom", - NetworkingMode::Macvtap => "macvtap", - } -} - fn networking_backend_name(mode: NetworkingMode) -> &'static str { match mode { NetworkingMode::Bridge => "tap_bridge", @@ -62,7 +53,7 @@ fn interfaces_to_proto( .map(|(index, networking)| { let mac = mac_address_for_vm_index(vm_id, &networking.mac_prefix_bytes(), index); pb::NetworkInterfaceStatus { - mode: networking_mode_name(networking.nic.mode).into(), + mode: networking.nic.mode.as_str().into(), backend: networking_backend_name(networking.nic.mode).into(), mac, bridge_name: (networking.nic.mode == NetworkingMode::Bridge) @@ -106,7 +97,7 @@ pub(crate) fn networking_to_proto(networking: &NicNetworking) -> pb::NetworkingC mode: if networking.inherit_mode { String::new() } else { - networking_mode_name(networking.mode).into() + networking.mode.as_str().into() }, bridge_name: if pins_backend && networking.mode == NetworkingMode::Bridge { networking.bridge.clone() @@ -207,6 +198,7 @@ impl VmInfo { .port_map .iter() .map(|mapping| pb::PortMapping { + nic_index: mapping.nic_index.map(|index| index as u32), protocol: mapping.protocol.as_str().into(), host_address: mapping.address.to_string(), host_port: mapping.from as u32, diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index d7479abc4..9ec20a115 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -357,11 +357,6 @@ pub struct CvmConfig { pub qemu_pci_hole64_size: u64, /// QEMU hotplug_off pub qemu_hotplug_off: bool, - /// Path to `qemu-bridge-helper`, used to attach an unprivileged TAP to a - /// host bridge. Empty probes the known distribution locations. - #[serde(default)] - pub qemu_bridge_helper: String, - /// TDX attestation/hash scheme policy. `legacy` keeps the existing /// digest.txt measurement path; `lite` opts into split measurement CBOR; /// `auto` selects `legacy` for @@ -772,16 +767,6 @@ impl Config { (1..=MAX_NET_QUEUES).contains(&self.cvm.max_net_queues), "cvm.max_net_queues must be between 1 and {MAX_NET_QUEUES}" ); - // The helper path is interpolated into QEMU's `-netdev` option list, - // which QEMU splits on ',' and '='. A path carrying either would not be - // passed through, it would end the option and start a bogus one, and the - // launch failure names neither this setting nor the file. Volume sources - // are rejected for the same reason. - anyhow::ensure!( - !self.cvm.qemu_bridge_helper.contains([',', '=']), - "cvm.qemu_bridge_helper must not contain ',' or '=': {}", - self.cvm.qemu_bridge_helper - ); anyhow::ensure!( !self .cvm @@ -906,10 +891,6 @@ fn validate_networking(networking: &Networking) -> Result<()> { !networking.nic.inherit_mode, "cvm.networking.inherit_mode is per-deployment state and cannot be set on the node default" ); - anyhow::ensure!( - networking.netd_interface.is_none(), - "cvm.networking.netd_interface is runtime state and cannot be set in configuration" - ); anyhow::ensure!( networking.device.is_empty(), "cvm.networking.device is runtime state and cannot be set in configuration" @@ -978,6 +959,19 @@ pub enum NetworkingMode { Macvtap, } +impl NetworkingMode { + /// The name this mode is written as in `vmm.toml`, in the RPC, and in + /// anything an operator reads. + pub fn as_str(self) -> &'static str { + match self { + NetworkingMode::User => "user", + NetworkingMode::Bridge => "bridge", + NetworkingMode::Custom => "custom", + NetworkingMode::Macvtap => "macvtap", + } + } +} + /// What a single NIC pins: the fields a deployment may name, a VM's manifest /// stores, and `GetInfo` reports back. /// @@ -1062,41 +1056,7 @@ pub struct Networking { // ── Custom fields ────────────────────────────────────────────── #[serde(default)] pub netdev: String, - // ── Runtime markers ──────────────────────────────────────────── - /// What netd built for this NIC, recorded when it was built. - /// - /// Runtime state, like `device`: resolution always clears it. Teardown - /// reads this rather than re-deriving it from node configuration, because - /// an operator may change `network_filter.mode` or `max_net_queues` while - /// the VM runs, and what has to be removed is what was created. - #[serde(default, skip_serializing_if = "NetdInterface::is_none")] - pub netd_interface: NetdInterface, -} - -/// The host interface netd created for a NIC, if any. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum NetdInterface { - /// netd was not involved: user mode, custom mode, or a bridge NIC that - /// QEMU's own bridge helper attaches. - #[default] - None, - /// netd created the interface and bound no libvirt nwfilter to it. - Unfiltered, - /// netd created the interface and bound a libvirt nwfilter to it, which - /// removal has to delete before the interface goes away. - Filtered, -} - -impl NetdInterface { - pub fn is_none(&self) -> bool { - matches!(self, NetdInterface::None) - } - - pub fn is_filtered(&self) -> bool { - matches!(self, NetdInterface::Filtered) - } } impl Networking { @@ -1449,25 +1409,21 @@ mod tests { .expect("default VMM config should parse") } - /// The two ownership markers are additive on disk: manifests and runtime - /// network snapshots written before they existed still load, and an entry - /// that carries neither serializes exactly as it used to. + /// The ownership marker is additive on disk: manifests and runtime network + /// snapshots written before it existed still load, and an entry that does + /// not carry it serializes exactly as it used to. #[test] fn ownership_markers_are_omitted_when_unset_and_default_when_absent() { let mut networking: Networking = serde_json::from_str(r#"{"mode":"bridge","bridge":"br0"}"#).unwrap(); assert!(!networking.nic.inherit_mode); - assert_eq!(networking.netd_interface, NetdInterface::None); let json = serde_json::to_string(&networking).unwrap(); assert!(!json.contains("inherit_mode"), "{json}"); - assert!(!json.contains("netd_interface"), "{json}"); networking.nic.inherit_mode = true; - networking.netd_interface = NetdInterface::Filtered; let json = serde_json::to_string(&networking).unwrap(); assert!(json.contains(r#""inherit_mode":true"#), "{json}"); - assert!(json.contains(r#""netd_interface":"filtered""#), "{json}"); assert_eq!( serde_json::from_str::(&json).unwrap(), networking @@ -1665,7 +1621,6 @@ mod networking_shape_tests { "vhost": false, "queues": 4, "inherit_mode": true, - "netd_interface": "filtered", }); // A resolved value keeps every field, at the same names as before. @@ -1694,26 +1649,6 @@ mod networking_shape_tests { assert!(stored.get("net").is_none()); } - /// The path is interpolated into QEMU's `-netdev` option list, which QEMU - /// splits on ',' and '='. A path carrying either would end the option and - /// start a bogus one, and the launch failure names neither the setting nor - /// the file. - #[test] - fn a_bridge_helper_path_cannot_end_the_qemu_option_it_sits_in() { - use rocket::figment::providers::Format as _; - let mut config: Config = rocket::figment::Figment::from( - rocket::figment::providers::Toml::string(DEFAULT_CONFIG), - ) - .extract() - .unwrap(); - config.cvm.qemu_bridge_helper = "/opt/qemu,helper".into(); - let error = config.validate().unwrap_err(); - assert!(error.to_string().contains("qemu_bridge_helper"), "{error}"); - - config.cvm.qemu_bridge_helper = "/usr/libexec/qemu-bridge-helper".into(); - config.validate().unwrap(); - } - /// The TOML section still deserializes through the flatten. #[test] fn the_node_section_still_parses_from_toml() { diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 17b93d194..d35864dc4 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -77,6 +77,48 @@ struct NetdArgs { /// Override the Unix socket configured in [netd]. #[arg(long)] socket: Option, + /// Inspect a running netd instead of starting one. + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum NetdCommand { + /// List every host interface netd holds. + /// + /// Answers the question a leak is made of -- whose is this interface -- + /// which deriving a name from an identity cannot. + List { + /// Only this VMM instance's interfaces. Defaults to every one netd + /// owns, including those it cannot attribute. + #[arg(long)] + instance: Option, + }, + /// Delete one interface by name. + /// + /// For what nothing else can name: an interface built before netd recorded + /// ownership, or by another netd, whose VM is gone. `netd list` shows these + /// with no instance and no VM, so nothing can derive the sweep that would + /// take them; an operator who can tell what they are says so here. + RemoveInterface { + /// The interface name, as `netd list` prints it. + name: String, + }, + /// Delete every interface netd holds for one VM. + /// + /// For a VM whose VMM will never ask again -- one whose directory was + /// deleted by hand, or whose instance is gone. A VMM sweeps its own VMs on + /// every stop and every removal, and keeps a removal pending until the + /// sweep lands; this is for when no VMM will ever run that sweep. + RemoveVm { + /// The `cvm.instance_id` of the VMM that created them. `netd list` + /// shows it. + #[arg(long)] + instance: String, + /// The VM's ID. + #[arg(long)] + vm: String, + }, } #[derive(ClapArgs)] @@ -192,6 +234,64 @@ async fn log_rotation_task(app: App) { } } +/// Client-side netd subcommands. Talks to the socket like the VMM does, so it +/// needs whatever the socket's permissions ask for and not root. +async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result<()> { + match command { + NetdCommand::List { instance } => { + let interfaces = netd::list(&config.socket, instance.as_deref().unwrap_or_default()) + .await + .context("failed to list netd interfaces")?; + println!( + "{:<16} {:<8} {:<24} {:<38} {:>3}", + "INTERFACE", "KIND", "INSTANCE", "VM", "NIC" + ); + let mut unattributed = 0; + for record in &interfaces { + if record.instance_id.is_none() { + unattributed += 1; + } + println!( + "{:<16} {:<8} {:<24} {:<38} {:>3}", + record.tap, + record.kind, + record.instance_id.as_deref().unwrap_or("-"), + record.vm_id.as_deref().unwrap_or("-"), + record + .nic_index + .map_or_else(|| "-".to_string(), |index| index.to_string()), + ); + } + println!(); + println!("{} interface(s)", interfaces.len()); + if unattributed > 0 { + // Not a fault to fix by hand: an interface built before netd + // recorded ownership, or by another netd, carries no record and + // gets one the next time its VM launches. + println!( + "{unattributed} carry no ownership record; `netd remove-interface` takes \ + one by name" + ); + } + Ok(()) + } + NetdCommand::RemoveInterface { name } => { + netd::remove_interface_named(&config.socket, name) + .await + .with_context(|| format!("failed to remove {name}"))?; + println!("removed {name}"); + Ok(()) + } + NetdCommand::RemoveVm { instance, vm } => { + let removed = netd::remove_all(&config.socket, instance, vm) + .await + .context("failed to remove the VM's interfaces")?; + println!("removed {removed} interface(s) for {vm}"); + Ok(()) + } + } +} + #[rocket::main] async fn main() -> Result<()> { { @@ -235,6 +335,9 @@ async fn main() -> Result<()> { .context("failed to load [cvm.network_filter] for netd")?, ); } + if let Some(command) = &netd_args.command { + return run_netd_command(&netd_config, command).await; + } return netd::serve(netd_config).await; } @@ -246,6 +349,7 @@ async fn main() -> Result<()> { // Preserve the existing startup validation. The broader static checks are // opt-in through `check-config` until they have seen wider deployment use. + netd::validate_instance_id(&config.cvm.instance_id)?; config .host_api .validate() diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 6f9a0905d..1c2835942 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -25,8 +25,9 @@ use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; use crate::app::{ - needs_swtpm, resolve_networking, validate_resolved_network, validate_resolved_networks, App, - AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, + mode_carries_ingress, needs_swtpm, resolve_networking, validate_resolved_network, + validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, + VmWorkDir, }; use crate::config::{CvmConfig, Networking, NetworkingMode, NicNetworking}; @@ -168,6 +169,62 @@ fn port_mappings_conflict(left: &PortMapping, right: &PortMapping) -> bool { || right.address.is_unspecified()) } +/// The backend each of a VM's NICs resolves to, with an empty list standing for +/// the node default's single NIC. +fn resolved_nic_modes( + networks: &[NicNetworking], + cvm_config: &CvmConfig, + vcpu: u32, +) -> Vec { + let node_default = [cvm_config.networking.nic.clone()]; + let requested = if networks.is_empty() { + &node_default[..] + } else { + networks + }; + requested + .iter() + .map(|networking| resolve_networking(networking, cvm_config, vcpu).nic.mode) + .collect() +} + +/// Rejects a mapping pinned to a NIC that cannot carry it. +/// +/// Both ways of getting that wrong are refused here, at deployment, where the +/// caller is present to be told: a NIC the VM does not have, and one whose +/// backend has no mechanism to publish a host port. Macvtap bypasses the host +/// bridge and a custom netdev is a string the VMM does not interpret, so a +/// mapping pinned to either used to resolve to that NIC and then fall out of +/// every branch that could act on it -- no `hostfwd=`, no netd request, and no +/// warning either. +fn validate_port_mapping_nics(mappings: &[PortMapping], modes: &[NetworkingMode]) -> Result<()> { + let nic_count = modes.len(); + for mapping in mappings { + let Some(index) = mapping.nic_index else { + continue; + }; + let Some(mode) = modes.get(index) else { + bail!( + "port mapping {} {}:{} names NIC {index}, but this VM has {nic_count}", + mapping.protocol.as_str(), + mapping.address, + mapping.from + ); + }; + if !mode_carries_ingress(*mode) { + bail!( + "port mapping {} {}:{} names NIC {index}, which is {} and cannot publish a host \ + port; use a user-mode or bridge NIC", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mode.as_str(), + ); + } + } + Ok(()) +} + fn validate_unique_port_mappings(mappings: &[PortMapping]) -> Result<()> { for (index, mapping) in mappings.iter().enumerate() { if mappings[..index] @@ -216,10 +273,16 @@ pub fn create_manifest_from_vm_config( protocol, from, to, + nic_index: p.nic_index.map(|index| index as usize), }) }) .collect::>>()?; validate_unique_port_mappings(&port_map)?; + let networks = networks_from_vm_config(&request, cvm_config)?; + validate_port_mapping_nics( + &port_map, + &resolved_nic_modes(&networks, cvm_config, request.vcpu), + )?; let app_id = match &request.app_id { Some(id) => id.strip_prefix("0x").unwrap_or(id).to_lowercase(), @@ -268,7 +331,7 @@ pub fn create_manifest_from_vm_config( no_tee: request.no_tee || simulated_tee.is_some(), simulated_tee, swtpm, - networks: networks_from_vm_config(&request, cvm_config)?, + networks, volumes, }) } @@ -865,6 +928,21 @@ impl VmmRpc for RpcHandler { async fn update_vm(self, request: UpdateVmRequest) -> Result { info!(vm_id = %request.id, "update_vm RPC called"); + // A VM being removed is not one to reconfigure. Before the lock, + // because removal holds it across the whole teardown and anything that + // only asked afterwards would wait that out in order to be told no. + self.app.refuse_if_removing(&request.id)?; + // Held from here rather than around the parts that touch the host, + // because everything below writes into the workdir -- the compose + // file first, the manifest last -- and `put_manifest` creates the + // directory it writes into. An update that resumed after a removal + // deleted that directory would recreate it holding nothing but a + // manifest: invisible to `list_vms`, unloadable at every start, and + // claiming the VM's netd interfaces against collection forever. + let _launch = self.app.launch_lock(&request.id).await; + // Again under the lock: removal can have claimed the VM while this + // waited for it. + self.app.refuse_if_removing(&request.id)?; let new_id = if !request.compose_file.is_empty() { // check the compose file is valid let _app_compose: AppCompose = @@ -918,6 +996,7 @@ impl VmmRpc for RpcHandler { protocol: p.protocol.parse().context("Invalid protocol")?, from: p.host_port.try_into().context("Invalid host port")?, to: p.vm_port.try_into().context("Invalid vm port")?, + nic_index: p.nic_index.map(|index| index as usize), }) }) .collect::>>()?; @@ -944,6 +1023,12 @@ impl VmmRpc for RpcHandler { let networks = networks_from_proto(&request.networks, &cvm)?; resolve_requested_networks(&networks, &cvm, manifest.vcpu)? }; + // Under the launch lock this whole call holds. Reading "not + // running" outside it and acting on the answer inside is the exact + // race the lock exists to close: a launch can start, prepare its + // interfaces and deploy QEMU in between, and the release would + // then delete the interfaces of a VM that is running -- silently, + // since QEMU stays up and the supervisor still reports it healthy. let is_running = self .app .supervisor @@ -951,15 +1036,24 @@ impl VmmRpc for RpcHandler { .await? .is_some_and(|info| info.state.status.is_running()); if !is_running { - let runtime_networks = vm_work_dir.runtime_networks(); - self.app - .remove_filtered_networks(&request.id, &runtime_networks) - .await - .context("failed to remove previous filtered networking")?; + self.app.release_vm_interfaces(&request.id).await; vm_work_dir.clear_runtime_networks()?; } manifest.networks = networks; } + // Both only when this request moved one of the two halves, and after + // both, since either half can move and the other still has to agree + // with it. A VM deployed before the node could answer for its ports + // must stay editable in every other respect: read-modify-write sends + // the whole configuration back, and refusing a memory change over a + // port mapping nobody touched -- or over a node default that changed + // under it -- would make the VM unmanageable rather than fixed. + if request.update_ports || request.update_networking { + validate_port_mapping_nics( + &manifest.port_map, + &resolved_nic_modes(&manifest.networks, &self.app.config.cvm, manifest.vcpu), + )?; + } let compose_file = fs::read_to_string(vm_work_dir.app_compose_path()) .context("failed to read app compose for swtpm decision")?; manifest.swtpm = needs_swtpm( @@ -1344,6 +1438,67 @@ mod tests { } } + fn pinned(nic_index: Option) -> PortMapping { + PortMapping { + address: "0.0.0.0".parse().unwrap(), + protocol: crate::config::Protocol::Tcp, + from: 443, + to: 443, + nic_index, + } + } + + /// Both ways of naming a NIC that cannot publish a port are refused where + /// the caller is present to be told. A pin to macvtap or to a custom + /// netdev used to resolve to that NIC and then fall out of every branch + /// that could act on it, so the port simply never appeared. + #[test] + fn a_pin_is_checked_against_the_backend_and_not_only_the_count() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.parent = "eth0".into(); + cvm.networking.nic.bridge = "br0".into(); + + let bridge_then_macvtap = vec![ + NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + ..NicNetworking::default() + }, + NicNetworking { + mode: NetworkingMode::Macvtap, + parent: "eth0".into(), + ..NicNetworking::default() + }, + ]; + let modes = resolved_nic_modes(&bridge_then_macvtap, &cvm, 2); + assert_eq!(modes, vec![NetworkingMode::Bridge, NetworkingMode::Macvtap]); + + // A bridge TAP is netd's, and netd does not forward host ports. + let error = validate_port_mapping_nics(&[pinned(Some(0))], &modes).unwrap_err(); + assert!(error.to_string().contains("bridge"), "{error}"); + + let error = validate_port_mapping_nics(&[pinned(Some(1))], &modes).unwrap_err(); + assert!(error.to_string().contains("macvtap"), "{error}"); + assert!( + error.to_string().contains("cannot publish a host port"), + "{error}" + ); + + let error = validate_port_mapping_nics(&[pinned(Some(2))], &modes).unwrap_err(); + assert!(error.to_string().contains("this VM has 2"), "{error}"); + + // An unpinned mapping is never refused here: where it lands is + // resolved at launch, from the topology in force then. + validate_port_mapping_nics(&[pinned(None)], &modes).unwrap(); + + // An empty list is the node default's one NIC, resolved the same way a + // launch would resolve it rather than assumed to be user mode. + cvm.networking.nic.mode = NetworkingMode::Macvtap; + let modes = resolved_nic_modes(&[], &cvm, 2); + assert_eq!(modes, vec![NetworkingMode::Macvtap]); + assert!(validate_port_mapping_nics(&[pinned(Some(0))], &modes).is_err()); + } + #[test] fn create_without_networking_persists_following_default() { let manifest = @@ -1634,6 +1789,9 @@ mod tests { .expect("a named backend is an override"); } + /// Deliberately restated rather than calling `NetworkingMode::as_str`: the + /// test below checks that what `GetInfo` reports is accepted back, and a + /// helper that shares the production mapping could only ever agree with it. fn networking_mode_name_for_test(mode: NetworkingMode) -> &'static str { match mode { NetworkingMode::Bridge => "bridge", diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 6a760d1eb..aca287a9c 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -5,6 +5,7 @@ //! Small privileged broker for TAP creation and libvirt nwfilter bindings. use std::{ + collections::HashSet, fs::{File, OpenOptions, Permissions}, io::Write as _, os::{ @@ -43,7 +44,20 @@ const LOCK_PATH: &str = "/run/lock/dstack-netd.lock"; /// Upper bound on TAP queue pairs netd will create. Mirrors the VMM's own cap /// so a malformed request cannot ask the kernel for an unbounded device. const MAX_QUEUES: u32 = 64; - +/// Highest NIC index an identity may name. Also the width of the space a +/// whole-VM sweep has to enumerate, since it derives names instead of reading a +/// record. +const MAX_NIC_INDEX: usize = 255; +/// The interface names netd may create. Reserved: anything matching it is +/// netd's to delete, and nothing else on the host may take one. +const TAP_PREFIX: &str = "dt"; +/// Hex characters of digest in a TAP name, after [`TAP_PREFIX`]. +const TAP_DIGEST_CHARS: usize = 12; +/// Version tag on the ownership record. Present so a later format can be told +/// from this one rather than mis-parsed as it. +const ALIAS_PREFIX: &str = "dstack1"; +/// What the kernel stores in an interface alias, minus the terminator. +const MAX_IFALIAS: usize = 255; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InterfaceIdentity { pub instance_id: String, @@ -69,6 +83,37 @@ pub struct PrepareBridgeRequest { /// rejects a device whose `IFF_MULTI_QUEUE` state differs from its own /// `queues=` argument, so this must match the launch exactly. pub queues: u32, + /// The VM's working directory on the host, for logs and diagnostics. + /// + /// Untrusted and never read for a decision: any process that can reach the + /// socket can assert anything here. It is carried so an operator reading + /// netd's log can get from an opaque TAP name back to the VM that asked for + /// it without going through the VMM. + #[serde(default)] + pub workdir: String, +} + +/// One host resource netd holds. +/// +/// `instance_id` and `vm_id` are absent when the interface carries no record +/// that checks out: built by a netd too old to write one, by a third-party +/// netd, or by this one in the instant between creating the interface and +/// recording it. Absent is not "nobody's" -- it is "not known to be anybody's", +/// which is a materially different thing to a collection. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InterfaceRecord { + pub tap: String, + /// `"tap"`, `"macvtap"`, or `"binding"` for an nwfilter binding whose + /// interface is already gone. A binding outlives the interface it was + /// bound to, so a collection that only looked at interfaces would leave + /// the one piece of state that survives them. + pub kind: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub instance_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vm_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nic_index: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -84,6 +129,10 @@ pub struct PrepareMacvtapRequest { /// queues; QEMU then opens the character device once per queue. #[serde(default)] pub queues: u32, + /// The VM's working directory on the host. Informational only; see + /// [`PrepareBridgeRequest::workdir`]. + #[serde(default)] + pub workdir: String, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -94,10 +143,43 @@ pub enum Request { Remove { #[serde(flatten)] identity: InterfaceIdentity, - /// Whether this interface was created with an nwfilter binding. - /// Macvtap TAPs never carry one, and removal detects them rather than - /// trusting this field. - filtered: bool, + }, + /// Delete every interface netd holds for one VM. + /// + /// Teardown by identity can only reach the NIC indices its caller still has + /// a record of, and that record is written after the interface exists: a + /// VMM killed in between leaves a TAP nothing on disk points at. A manifest + /// that lost a NIC leaves the same thing behind. Both are found here + /// without a record, because every name netd can produce for a VM is + /// derivable from its identity. + RemoveAll { + instance_id: String, + vm_id: String, + }, + /// Everything netd holds, so that an operator can see the host's + /// interfaces without being told what to look for. + /// + /// Deriving a name answers "where is this VM's interface". It cannot + /// answer "whose is this interface", which is the question a leak is made + /// of: a VM whose directory was deleted, a VMM instance that was + /// decommissioned, an interface built by a netd that has since been + /// upgraded. Enumeration answers it. + List { + /// Only interfaces recorded as this instance's. Empty lists every one + /// netd owns, whatever it is recorded as and whether or not it is. + #[serde(default)] + instance_id: String, + }, + /// Delete one interface by name. + /// + /// For what nothing else can reach: an interface built before netd recorded + /// ownership, or by another netd, whose VM is gone. Nothing can attribute + /// it, so nothing can decide about it -- but an operator looking at + /// `list` can, and this is how they say so. Guarded the same way every + /// other removal is: the name must be one netd could have created, and the + /// device must be one netd creates. + RemoveInterface { + tap: String, }, /// Verify a deterministic TAP and binding for operations and integration /// diagnostics. The VMM startup path uses Prepare rather than Check. @@ -120,26 +202,73 @@ struct Response { /// between "one queue was requested" and "this netd ignored the request". #[serde(default, skip_serializing_if = "Option::is_none")] queues: Option, + /// How many interfaces a whole-VM sweep deleted. + #[serde(default, skip_serializing_if = "Option::is_none")] + removed: Option, + /// For a listing, everything netd holds; for a collection, what it took, + /// or would take on a dry run. Absent, rather than empty, from a netd that + /// cannot enumerate: "I hold nothing" and "I cannot say" are answers a + /// collection must not confuse. + #[serde(default, skip_serializing_if = "Option::is_none")] + interfaces: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, } -/// What netd built, echoed back so the caller can verify it matches the -/// request before handing the interface to QEMU. -struct Prepared { - tap: String, - device: Option, - queues: Option, +/// What one request produced. +/// +/// An enum rather than a struct of options, because the shapes do not overlap: +/// a prepare names an interface, a sweep counts them, a listing enumerates. +/// One flat [`Response`] still carries all of them on the wire, so a netd that +/// grows an operation stays readable to a caller that does not know it. +enum Outcome { + /// One interface, named because the caller hands that name to QEMU. + Interface { + tap: String, + device: Option, + queues: Option, + }, + /// A sweep names no single interface, so it reports how many it deleted. + Swept { + removed: usize, + }, + Listed(Vec), } -impl Prepared { +impl Outcome { fn tap(tap: String) -> Self { - Self { + Self::Interface { tap, device: None, queues: None, } } + + fn into_response(self) -> Response { + let mut response = Response { + ok: true, + tap: None, + device: None, + queues: None, + removed: None, + interfaces: None, + error: None, + }; + match self { + Self::Interface { + tap, + device, + queues, + } => { + response.tap = Some(tap); + response.device = device; + response.queues = queues; + } + Self::Swept { removed } => response.removed = Some(removed), + Self::Listed(interfaces) => response.interfaces = Some(interfaces), + } + response + } } pub fn tap_name(identity: &InterfaceIdentity) -> String { @@ -148,7 +277,82 @@ pub fn tap_name(identity: &InterfaceIdentity) -> String { identity.instance_id, identity.vm_id, identity.nic_index ); let digest = Sha256::digest(input.as_bytes()); - format!("dt{}", hex::encode(&digest[..6])) + format!( + "{TAP_PREFIX}{}", + hex::encode(&digest[..TAP_DIGEST_CHARS / 2]) + ) +} + +/// The ownership record netd writes onto every interface it creates. +/// +/// The record lives on the resource, so it has exactly the resource's +/// lifetime. A file under `/run` would be a second thing to keep in step with +/// the first, and the failure this whole path exists to fix is precisely a +/// record that got out of step: written after the interface, lost with the +/// directory, and unreadable to anything but the process that wrote it. +/// +/// Never trusted as *authority*. Anything that can reach this socket can also +/// name an identity, and the interface name is a digest of that identity -- +/// so a record is believed only when re-deriving the name from it reproduces +/// the name it is written on. Ambiguity (a separator inside an identity), +/// truncation, and forgery all fail that check and land in the same bucket as +/// no record at all, which is the bucket handled conservatively. +pub fn interface_alias(identity: &InterfaceIdentity) -> String { + format!( + "{ALIAS_PREFIX}:{}:{}:{}", + identity.nic_index, identity.instance_id, identity.vm_id + ) +} + +/// The identity an interface claims, if the claim checks out. +/// +/// `nic_index` first, so the two free-form fields are the last two and a +/// `vm_id` containing the separator still parses. An `instance_id` containing +/// one does not, and is refused at prepare rather than mis-parsed here. +pub fn owner_of(tap: &str, alias: &str) -> Option { + // `trim_end_matches`, not `trim`: sysfs adds a newline, and a `vm_id` + // whose own trailing whitespace were trimmed off here would re-derive a + // name that is not the one it is on, making the interface permanently + // unattributable -- never collected, only removable by hand. + let rest = alias + .trim_end_matches(['\n', '\r']) + .strip_prefix(ALIAS_PREFIX)? + .strip_prefix(':')?; + let (nic_index, rest) = rest.split_once(':')?; + let (instance_id, vm_id) = rest.split_once(':')?; + let identity = InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + nic_index: nic_index.parse().ok()?, + }; + // The name is the proof. A record that does not reproduce it describes + // some other interface, or nothing. + (tap_name(&identity) == tap).then_some(identity) +} + +/// Whether this name is one netd can have created. See [`TAP_PREFIX`]. +pub fn is_managed_name(interface: &str) -> bool { + let Some(digest) = interface.strip_prefix(TAP_PREFIX) else { + return false; + }; + digest.len() == TAP_DIGEST_CHARS + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Rejects an instance ID no interface could be recorded as belonging to. +/// +/// At startup rather than at the first launch. The VMM derives one that is +/// always valid; an operator who configured their own learns here rather than +/// from the first VM that fails to get a NIC. +pub fn validate_instance_id(instance_id: &str) -> Result<()> { + validate_identity(&InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: "0".repeat(64), + nic_index: MAX_NIC_INDEX, + }) + .context("invalid cvm.instance_id") } pub fn instance_id(configured: &str, run_path: &Path) -> String { @@ -181,23 +385,96 @@ impl std::fmt::Display for Unreachable { impl std::error::Error for Unreachable {} /// Whether this error means netd was never reached. +/// +/// `downcast_ref` rather than a walk over `chain()`: a marker attached with +/// `context` is not a link in the source chain, it is the context *of* a link, +/// and `chain()` yields the wrapper rather than the marker inside it. Asking +/// the chain therefore always answered no -- which made every "an unreachable +/// netd is not a failure" branch in this crate unreachable itself. pub fn is_unreachable(error: &anyhow::Error) -> bool { - error.chain().any(|cause| cause.is::()) + error.downcast_ref::().is_some() } pub async fn request(socket: &Path, request: &Request) -> Result { + let response = exchange(socket, request).await?; + if response.tap.as_deref().unwrap_or_default().is_empty() { + bail!("netd response omitted TAP name"); + } + Ok(PreparedInterface { + device: response.device, + queues: response.queues, + }) +} + +/// Deletes every interface netd holds for one VM, returning how many there +/// were. See [`Request::RemoveAll`]. +/// +/// A netd that answers without a count did not sweep. Reading that as zero +/// would report a netd that cannot do this as a VM that had nothing to remove, +/// which is a netd that cannot do this reported as a VM that had nothing to +/// remove -- so it is an error here. +pub async fn remove_all(socket: &Path, instance_id: &str, vm_id: &str) -> Result { + let request = Request::RemoveAll { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + }; + exchange(socket, &request) + .await? + .removed + .context("netd answered a sweep without saying what it removed") +} + +/// Deletes one interface by name. See [`Request::RemoveInterface`]. +pub async fn remove_interface_named(socket: &Path, tap: &str) -> Result<()> { + let request = Request::RemoveInterface { + tap: tap.to_string(), + }; + exchange(socket, &request).await.map(|_| ()) +} + +/// Everything netd holds, optionally narrowed to one VMM instance. See +/// [`Request::List`]. +pub async fn list(socket: &Path, instance_id: &str) -> Result> { + let request = Request::List { + instance_id: instance_id.to_string(), + }; + exchange(socket, &request) + .await? + .interfaces + .context("netd answered a listing without one") +} + +async fn exchange(socket: &Path, request: &Request) -> Result { let operation = match request { Request::PrepareBridge(_) => "prepare_bridge", Request::PrepareMacvtap(_) => "prepare_macvtap", Request::Remove { .. } => "remove", + Request::RemoveAll { .. } => "remove_all", + Request::List { .. } => "list", + Request::RemoveInterface { .. } => "remove_interface", Request::Check { .. } => "check", }; let exchange = async { - let mut stream = UnixStream::connect(socket) - .await - .map_err(anyhow::Error::from) - .context(Unreachable) - .with_context(|| format!("failed to connect to netd at {}", socket.display()))?; + let mut stream = UnixStream::connect(socket).await.map_err(|error| { + // Only the two errnos that mean "nothing is listening". A socket + // the VMM's user cannot open (`EACCES`, the default `0660` on a + // root-owned socket) or a VMM out of descriptors would otherwise + // read as an absent netd, and every caller that treats absence as + // "nothing to do here" would skip its work at `debug!` -- leaking + // interfaces on a host whose netd is running fine, while telling + // the operator to go start one. + let absent = matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::ConnectionRefused + ); + let error = anyhow::Error::from(error) + .context(format!("failed to connect to netd at {}", socket.display())); + if absent { + error.context(Unreachable) + } else { + error + } + })?; let message = serde_json::to_vec(request)?; if message.len() as u64 > MAX_MESSAGE_SIZE { bail!("netd request is too large"); @@ -220,11 +497,7 @@ pub async fn request(socket: &Path, request: &Request) -> Result Result<()> { None => bind_listener(&config)?, }; info!(address = ?listener.local_addr()?, "netd listening"); + let config = std::sync::Arc::new(config); loop { let (mut stream, _) = listener.accept().await?; - // This timeout bounds async socket reads and writes. handle_request is - // synchronous, so helper execution is bounded separately by - // COMMAND_TIMEOUT rather than preempted by this future timeout. - match timeout(CONNECTION_TIMEOUT, serve_connection(&config, &mut stream)).await { - Ok(Ok(())) => {} - Ok(Err(error)) => warn!(%error, "netd connection failed"), - Err(_) => warn!("netd connection timed out"), - } + // One task per connection, so a request that takes minutes does not + // stop the next one from being *accepted*. + // + // Serialization still holds where it matters, and holds where it is + // actually stated: `handle_request` takes the operation lock, which is + // an flock and blocks between two open descriptions in one process + // just as it does between processes. What a single-connection accept + // loop added on top of that was head-of-line blocking -- a caller + // asking netd a question it answers in microseconds waited for whatever + // netd happened to be doing, and gave up believing netd was not there. + // A collection can run for twenty seconds and a `virsh` for thirty, so + // this was not a corner: it made a busy netd indistinguishable from an + // absent one, and teardown skips an absent netd. + let config = config.clone(); + tokio::spawn(async move { + if let Err(error) = serve_connection(&config, &mut stream).await { + warn!(%error, "netd connection failed"); + } + }); } } @@ -287,19 +572,46 @@ fn bind_listener(config: &NetdConfig) -> Result { Ok(listener) } -async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Result<()> { +/// Serves one connection. +/// +/// The timeouts bound the socket reads and writes and nothing else. They used +/// to wrap the whole exchange, which read as a bound on the request but was +/// not one: `handle_request` is synchronous and shells out, so the timeout +/// could not cancel it -- it only threw away the answer to work that went on +/// running. A caller has its own deadline and will have gone by then; what it +/// cost was netd's own log, which reported "timed out" for a request it in +/// fact completed. Helper execution is bounded where it can be, by +/// COMMAND_TIMEOUT per invocation. +async fn serve_connection( + config: &std::sync::Arc, + stream: &mut UnixStream, +) -> Result<()> { // Access is authorized by the Unix socket's owner, group, and mode. Any // process that can connect is trusted with the complete netd protocol. - let outcome = match read_request(stream).await { - // A peer that connects and closes without sending is the VMM's - // reachability check: netd that died leaves its socket behind, so the - // VMM connects to tell the two apart. Answering that with a parse error - // and a warning would fill the log with reports of it working. + let request = timeout(CONNECTION_TIMEOUT, read_request(stream)) + .await + .context("timed out reading a netd request")?; + let outcome = match request { + // A peer that connects and closes without sending is asking whether + // anything is listening: netd that died leaves its socket behind. + // Answering that with a parse error and a warning would fill the log + // with reports of it working. Ok(None) => { debug!("netd liveness probe"); return Ok(()); } - Ok(Some(request)) => handle_request(config, request), + Ok(Some(request)) => { + // `handle_request` shells out to `ip` and `virsh` and waits on the + // operation lock, so it can block for as long as those take. On a + // runtime worker that would stall every other connection's reads + // and writes, which is the head-of-line blocking this daemon just + // stopped having. + let config = config.clone(); + match tokio::task::spawn_blocking(move || handle_request(&config, request)).await { + Ok(outcome) => outcome, + Err(error) => Err(anyhow::anyhow!("netd worker failed: {error}")), + } + } // A request that arrived but could not be understood still gets an // answer. A VMM newer than this netd sends operations it does not // know, and "unknown variant `prepare_foo`" is what tells the operator @@ -307,13 +619,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul Err(error) => Err(error), }; let response = match outcome { - Ok(prepared) => Response { - ok: true, - tap: Some(prepared.tap), - device: prepared.device, - queues: prepared.queues, - error: None, - }, + Ok(outcome) => outcome.into_response(), Err(error) => { warn!(%error, "netd request failed"); Response { @@ -321,13 +627,19 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul tap: None, device: None, queues: None, + removed: None, + interfaces: None, error: Some(format!("{error:#}")), } } }; let encoded = serde_json::to_vec(&response)?; - stream.write_all(&encoded).await?; - stream.shutdown().await?; + timeout(CONNECTION_TIMEOUT, async { + stream.write_all(&encoded).await?; + stream.shutdown().await + }) + .await + .context("timed out answering a netd request")??; Ok(()) } @@ -349,7 +661,7 @@ async fn read_request(stream: &mut UnixStream) -> Result> { .context("invalid netd request") } -fn handle_request(config: &NetdConfig, request: Request) -> Result { +fn handle_request(config: &NetdConfig, request: Request) -> Result { let libvirt_uri = config.libvirt_uri.as_str(); let _lock = OperationLock::acquire()?; match request { @@ -359,11 +671,32 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { Request::PrepareMacvtap(request) => { prepare_macvtap(libvirt_uri, &request, config.filter_policy()) } - Request::Remove { identity, filtered } => { + Request::List { instance_id } => { + Ok(Outcome::Listed(list_interfaces(libvirt_uri, &instance_id))) + } + Request::RemoveAll { instance_id, vm_id } => { + let removed = sweep_vm_interfaces(libvirt_uri, &instance_id, &vm_id)?; + Ok(Outcome::Swept { removed }) + } + Request::RemoveInterface { tap } => { + if !is_managed_name(&tap) { + bail!("{tap} is not a name netd could have created"); + } + remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; + Ok(Outcome::tap(tap)) + } + Request::Remove { identity } => { validate_identity(&identity)?; let tap = tap_name(&identity); - remove_interface(libvirt_uri, &tap, binding_cleanup(filtered))?; - Ok(Prepared::tap(tap)) + // Best effort, whatever the caller says was built. The strict rule + // exists for prepare, where a binding left at the name would block + // the one about to be created; at removal nothing is about to take + // the name, and failing here leaves the interface itself up on the + // bridge rather than just a binding libvirt will hand back on its + // next listing. It also makes this agree with the whole-VM sweep, + // which has always been best effort. + remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; + Ok(Outcome::tap(tap)) } Request::Check { identity, filtered } => { validate_identity(&identity)?; @@ -376,7 +709,7 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { if filtered && !is_macvtap(&tap) { virsh(libvirt_uri, &["nwfilter-binding-dumpxml", &tap], None)?; } - Ok(Prepared::tap(tap)) + Ok(Outcome::tap(tap)) } } } @@ -385,7 +718,7 @@ fn prepare_macvtap( libvirt_uri: &str, request: &PrepareMacvtapRequest, filter: &NetworkFilterConfig, -) -> Result { +) -> Result { let identity = &request.identity; let parent = request.parent.as_str(); let qemu_uid = request.qemu_uid; @@ -431,6 +764,7 @@ fn prepare_macvtap( add.extend_from_slice(&["type", "macvtap", "mode", mode]); ip(&add)?; let result = (|| { + set_alias(&tap, identity)?; let ifindex = std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifindex")) .context("failed to read macvtap ifindex")?; @@ -455,7 +789,7 @@ fn prepare_macvtap( match result { Ok(device) => { info!(%tap, %parent, %mode, %device, %queues, "prepared macvtap"); - Ok(Prepared { + Ok(Outcome::Interface { tap, device: Some(device), queues: Some(queues), @@ -500,13 +834,25 @@ fn prepare_bridge( libvirt_uri: &str, request: &PrepareBridgeRequest, filter: &NetworkFilterConfig, -) -> Result { +) -> Result { validate_prepare_bridge(request, filter)?; let filtered = request.filtered; let tap = tap_name(&request.identity); // A failed VMM start may leave a deterministic resource behind. Replacing // it makes prepare idempotent without accepting a caller-selected TAP. - remove_interface(libvirt_uri, &tap, binding_cleanup(filtered))?; + // A binding outlives the interface it was bound to and TAP names are + // derived, so the same name comes back: clear whatever is there. Insist + // only when this prepare is about to create a replacement libvirt would + // refuse as a duplicate. + remove_interface( + libvirt_uri, + &tap, + if filtered { + BindingCleanup::Required + } else { + BindingCleanup::BestEffort + }, + )?; let uid = request.qemu_uid.to_string(); let queues = validate_queues(request.queues)?; @@ -519,6 +865,10 @@ fn prepare_bridge( add.extend_from_slice(&["user", &uid]); ip(&add)?; let result = (|| { + // Before anything else it could fail at. An interface that exists + // without a record is one nothing can attribute, and the window in + // which that is true is the window a crash turns permanent. + set_alias(&tap, &request.identity)?; ip(&["link", "set", "dev", &tap, "master", &request.bridge])?; if filtered { let xml = binding_xml(request, &tap, filter); @@ -536,10 +886,13 @@ fn prepare_bridge( return Err(error); } info!(%tap, bridge = %request.bridge, %filtered, %queues, "prepared TAP"); - Ok(Prepared { + Ok(Outcome::Interface { tap, device: None, queues: Some(queues), + // This netd builds interfaces; it is not the host's forwarder. Saying + // nothing here is what tells the caller that, so ports it asked for are + // reported as unmet rather than assumed done. }) } @@ -555,11 +908,160 @@ enum BindingCleanup { /// be running at all, and a stale binding left by an earlier, filtered /// interface at this name is still worth clearing when it is. BestEffort, + /// The caller has already decided about the binding. Used by a pass over + /// many interfaces, which asks libvirt once about all of them rather than + /// once per interface. + Skip, +} + +/// Deletes every interface a VM could hold, by deriving each name rather than +/// consulting a record. +/// +/// `validate_identity` caps the NIC index, so the whole space a VM can occupy +/// is enumerable: 256 names, each a `stat` that usually misses. Cleanup is +/// best-effort about bindings -- nothing is about to take these names, and a +/// node running unfiltered TAPs need not have libvirtd at all. +/// +/// A name with no interface is not skipped. An nwfilter binding outlives the +/// TAP it was bound to, so the one state teardown must not leave behind is +/// exactly the one a `/sys/class/net` check cannot see: the per-name Remove +/// this replaced deleted the binding unconditionally, and a sweep that reaches +/// less than the thing it replaced is not a sweep. Those names are decided +/// against a single listing, because the whole point of enumerating a bounded +/// space is that deciding one name stays cheap. +fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Result { + let identity = InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + nic_index: 0, + }; + validate_identity(&identity)?; + let bindings = existing_bindings(libvirt_uri); + // Not `bindings.is_some()`. A listing that could not be produced says + // nothing about whether a *deletion* will work, and reading it as "libvirt + // is down, skip the bindings" would mean a node whose listing breaks for + // any reason silently stops cleaning up bindings at all -- which is worse + // than the per-name asking this listing exists to avoid. The pass finds + // out by trying, once. + let mut libvirt = true; + let mut removed = 0; + let mut first_error = None; + for nic_index in 0..=MAX_NIC_INDEX { + let tap = tap_name(&InterfaceIdentity { + nic_index, + ..identity.clone() + }); + let present = Path::new("/sys/class/net").join(&tap).exists(); + // A pass gets one answer about libvirt, not one per interface. Asking + // again after it has failed is how a hung `libvirtd` turns a bounded + // collection into an unbounded one. + let wanted = present || bindings.as_ref().is_some_and(|held| held.contains(&tap)); + if libvirt && wanted && !is_macvtap(&tap) { + if let Err(error) = delete_binding(libvirt_uri, &tap) { + warn!(%tap, %error, "failed to remove an nwfilter binding"); + libvirt = false; + first_error.get_or_insert(error); + } else if !present { + info!(%tap, %vm_id, "removed orphaned nwfilter binding"); + } + } + if !present { + continue; + } + // Keep going after a failure. Stopping at the first one would leave the + // rest of a VM's interfaces behind over one that is stuck. + match remove_interface(libvirt_uri, &tap, BindingCleanup::Skip) { + Err(error) => { + warn!(%tap, %error, "failed to remove interface"); + first_error.get_or_insert(error); + } + Ok(()) => { + info!(%tap, %vm_id, "removed interface"); + removed += 1; + } + } + } + match first_error { + Some(error) => Err(error).context("failed to remove every interface for this VM"), + None => Ok(removed), + } +} + +/// Every resource netd owns, read off the host rather than out of a record. +/// +/// Ownership is the reserved name plus the kernel's own answer about what kind +/// of device it is; attribution is the interface's alias, checked by +/// re-deriving the name from it. A listing never fails for want of libvirt: on +/// a node that does not filter, `libvirtd` need not be running, and an +/// interface inventory that refused to be produced without it would be +/// unavailable exactly where unfiltered TAPs live. +fn list_interfaces(libvirt_uri: &str, instance_id: &str) -> Vec { + let bindings = existing_bindings(libvirt_uri); + let mut records = Vec::new(); + let mut seen = HashSet::new(); + if let Ok(entries) = std::fs::read_dir("/sys/class/net") { + for entry in entries.flatten() { + let Ok(tap) = entry.file_name().into_string() else { + continue; + }; + if !is_managed_name(&tap) { + continue; + } + let kind = if is_macvtap(&tap) { + "macvtap" + } else if is_tuntap(&tap) { + "tap" + } else { + // The name is netd's to use, but this is not a device netd + // creates. Listing it would invite a caller to delete it. + continue; + }; + let alias = + std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifalias")) + .unwrap_or_default(); + let owner = owner_of(&tap, &alias); + seen.insert(tap.clone()); + records.push(InterfaceRecord { + kind: kind.to_string(), + nic_index: owner.as_ref().map(|identity| identity.nic_index), + instance_id: owner.as_ref().map(|identity| identity.instance_id.clone()), + vm_id: owner.map(|identity| identity.vm_id), + tap, + }); + } + } + // A binding outlives its interface, and an interface is the only thing that + // carries a record, so an orphaned binding can never be attributed. It is + // still netd's: nothing else creates a binding at one of these names. + for name in bindings.into_iter().flatten() { + if !seen.contains(&name) { + records.push(InterfaceRecord { + tap: name, + kind: "binding".to_string(), + instance_id: None, + vm_id: None, + nic_index: None, + }); + } + } + if !instance_id.is_empty() { + records.retain(|record| record.instance_id.as_deref() == Some(instance_id)); + } + records.sort_by(|left, right| left.tap.cmp(&right.tap)); + records } fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Result<()> { let macvtap = is_macvtap(tap); if Path::new("/sys/class/net").join(tap).exists() { + // The name is 48 bits of SHA-256, so a collision is not the worry. A + // caller asserting an identity that happens to derive to some + // pre-existing device is: netd runs as root and `ip link delete` does + // not ask what it is deleting. netd creates exactly two kinds of + // device, and the kernel publishes an attribute unique to each. + if !macvtap && !is_tuntap(tap) { + bail!("refusing to delete {tap}: it is neither a tun/tap nor a macvtap device"); + } let _ = ip(&["link", "set", "dev", tap, "down"]); } // A macvtap interface never carries a binding. Anything else might: this @@ -567,6 +1069,7 @@ fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Re // outlives the interface. if !macvtap { match cleanup { + BindingCleanup::Skip => {} BindingCleanup::Required => delete_binding(libvirt_uri, tap)?, BindingCleanup::BestEffort => { // netd refuses to start without virsh, so the binary is always @@ -585,6 +1088,24 @@ fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Re Ok(()) } +/// Records who an interface belongs to, on the interface. See +/// [`interface_alias`]. +fn set_alias(tap: &str, identity: &InterfaceIdentity) -> Result<()> { + let alias = interface_alias(identity); + ip(&["link", "set", "dev", tap, "alias", &alias]) + .with_context(|| format!("failed to record ownership on {tap}")) +} + +/// Whether this is a tun/tap device. `tun_flags` is published by the tun +/// driver and by nothing else, so its presence is the kernel's own answer -- +/// as `macvtap/` is for the other kind of device netd creates. +fn is_tuntap(interface: &str) -> bool { + Path::new("/sys/class/net") + .join(interface) + .join("tun_flags") + .exists() +} + fn is_macvtap(interface: &str) -> bool { Path::new("/sys/class/net") .join(interface) @@ -595,8 +1116,9 @@ fn is_macvtap(interface: &str) -> bool { /// Deletes an interface's nwfilter binding, if it has one. /// /// Goes through the same `COMMAND_TIMEOUT`-bounded helper as every other virsh -/// call. netd's accept loop is strictly serialized, so an unbounded call here -/// would let one unreachable libvirt stall every other VM's prepare and remove. +/// call. Every mutating operation holds the operation lock, so an unbounded +/// call here would let one unreachable libvirt stall every other VM's prepare +/// and remove behind it. fn delete_binding(uri: &str, tap: &str) -> Result<()> { match virsh(uri, &["nwfilter-binding-delete", tap], None) { Ok(()) => Ok(()), @@ -694,20 +1216,26 @@ fn validate_identity(identity: &InterfaceIdentity) -> Result<()> { bail!("invalid {label}"); } } - if identity.nic_index > 255 { + if identity.nic_index > MAX_NIC_INDEX { bail!("NIC index is out of range"); } - Ok(()) -} - -/// A caller that knows a binding is there needs it gone; one that does not -/// still clears whatever it finds, without failing when libvirt is absent. -fn binding_cleanup(filtered: bool) -> BindingCleanup { - if filtered { - BindingCleanup::Required - } else { - BindingCleanup::BestEffort + // An identity that cannot be recorded on the interface is refused rather + // than built unattributed. A host resource nothing can name the owner of + // is the thing this whole path exists to stop producing, and the kernel's + // alias is the only place with the interface's exact lifetime to put it. + let alias = interface_alias(identity); + if alias.len() > MAX_IFALIAS { + bail!( + "identity is too long to record on the interface: {} bytes of {MAX_IFALIAS}", + alias.len() + ); + } + // The record puts the two free-form fields last, so only the first of them + // has to be unambiguous. + if identity.instance_id.contains(':') { + bail!("instance ID must not contain ':'"); } + Ok(()) } /// Normalizes a requested queue pair count. Zero means the caller did not ask @@ -767,13 +1295,47 @@ fn ip(args: &[&str]) -> Result<()> { } fn virsh(uri: &str, args: &[&str], stdin: Option<&[u8]>) -> Result<()> { + virsh_output(uri, args, stdin).map(|_| ()) +} + +fn virsh_output(uri: &str, args: &[&str], stdin: Option<&[u8]>) -> Result { let mut full_args = vec!["--connect", uri]; full_args.extend_from_slice(args); - run_command(VIRSH_PATH, &full_args, stdin) + run_command_with_timeout(VIRSH_PATH, &full_args, stdin, COMMAND_TIMEOUT) +} + +/// Every nwfilter binding libvirt holds at a name netd could have created. +/// +/// One call, so that a sweep can decide 256 names against a set instead of +/// asking libvirt 256 times. `None` means libvirt could not be asked at all, +/// which on a node running unfiltered TAPs is the normal state -- `virsh` must +/// be installed for netd to start, but `libvirtd` need not be running. +/// +/// The command has no machine-readable mode: it prints a two-line header and +/// then one binding per line, interface name first, and it accepts no options +/// at all -- `--name` is not one of them, and asking for it fails the whole +/// call. Narrowing to netd's own name space is what makes parsing a human +/// table safe: a header, a rule line, or a column that moves cannot produce a +/// `dt` name, and a binding at any other name is not netd's to reason about. +fn existing_bindings(uri: &str) -> Option> { + match virsh_output(uri, &["nwfilter-binding-list"], None) { + Ok(output) => Some( + output + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|name| is_managed_name(name)) + .map(str::to_string) + .collect(), + ), + Err(error) => { + debug!("could not list nwfilter bindings: {error:#}"); + None + } + } } fn run_command(program: &str, args: &[&str], stdin: Option<&[u8]>) -> Result<()> { - run_command_with_timeout(program, args, stdin, COMMAND_TIMEOUT) + run_command_with_timeout(program, args, stdin, COMMAND_TIMEOUT).map(|_| ()) } fn run_command_with_timeout( @@ -781,7 +1343,7 @@ fn run_command_with_timeout( args: &[&str], stdin: Option<&[u8]>, command_timeout: Duration, -) -> Result<()> { +) -> Result { let mut child = Command::new(program) .args(args) .stdin(if stdin.is_some() { @@ -813,7 +1375,7 @@ fn run_command_with_timeout( let error = String::from_utf8_lossy(&output.stderr); bail!("{} failed: {}", Path::new(program).display(), error.trim()); } - Ok(()) + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } fn require_executable(path: &str) -> Result<()> { @@ -837,6 +1399,163 @@ fn prepare_socket_path(socket: &Path) -> Result<()> { Ok(()) } +/// A netd that exists only to be talked to. +/// +/// The VMM's side of this protocol -- what it falls back to, what it refuses, +/// what it does when the answer is missing -- had no test at all, because +/// every path needed a privileged daemon. It does not: it needs something that +/// answers on a socket. This is that, scripted per behaviour, recording what +/// it was asked so a test can assert on the conversation rather than on its +/// effects. +#[cfg(test)] +pub(crate) mod testing { + use std::{ + path::{Path, PathBuf}, + sync::{Arc, Mutex}, + }; + + use serde_json::{json, Value}; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::UnixListener, + }; + + /// How the fake answers. + #[derive(Debug, Clone)] + pub(crate) enum Behavior { + /// Answers every listed operation with a plausible success, and + /// anything else with the error `serde` produces for an unknown one. + Handles(Vec), + /// An older netd: every operation this one added is an error. + Legacy, + } + + impl Behavior { + pub(crate) fn handling(operations: &[&str]) -> Self { + Self::Handles(operations.iter().map(|name| name.to_string()).collect()) + } + } + + pub(crate) struct FakeNetd { + _dir: tempfile::TempDir, + socket: PathBuf, + seen: Arc>>, + } + + impl FakeNetd { + pub(crate) fn spawn(behavior: Behavior) -> Self { + Self::spawn_holding(behavior, Vec::new()) + } + + /// A netd that holds these interfaces, whatever else it does. + pub(crate) fn spawn_holding(behavior: Behavior, interfaces: Vec) -> Self { + Self::start(behavior, interfaces) + } + + fn start(behavior: Behavior, interfaces: Vec) -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("netd.sock"); + let listener = UnixListener::bind(&socket).expect("bind"); + let seen = Arc::new(Mutex::new(Vec::new())); + let recorder = seen.clone(); + let interfaces = std::sync::Arc::new(interfaces); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let behavior = behavior.clone(); + let interfaces = interfaces.clone(); + let recorder = recorder.clone(); + tokio::spawn(async move { + let mut message = Vec::new(); + if stream.read_to_end(&mut message).await.is_err() { + return; + } + let Ok(request) = serde_json::from_slice::(&message) else { + return; + }; + recorder.lock().expect("poisoned").push(request.clone()); + let response = answer(&behavior, &interfaces, &request); + let _ = stream + .write_all(&serde_json::to_vec(&response).unwrap()) + .await; + let _ = stream.shutdown().await; + }); + } + }); + Self { + _dir: dir, + socket, + seen, + } + } + + pub(crate) fn socket(&self) -> &Path { + &self.socket + } + + /// Every request it was sent, in order. + pub(crate) fn seen(&self) -> Vec { + self.seen.lock().expect("poisoned").clone() + } + + pub(crate) fn operations(&self) -> Vec { + self.seen() + .iter() + .map(|request| request["operation"].as_str().unwrap_or("?").to_string()) + .collect() + } + } + + fn answer(behavior: &Behavior, interfaces: &[Value], request: &Value) -> Value { + let operation = request["operation"].as_str().unwrap_or_default(); + let operations = match behavior { + Behavior::Legacy => { + return match operation { + // What the real thing answers for an operation it knows. + "prepare_bridge" | "prepare_macvtap" | "remove" | "check" => { + json!({"ok": true, "tap": "dtdeadbeef00"}) + } + other => json!({ + "ok": false, + "error": format!("invalid netd request: unknown variant `{other}`"), + }), + }; + } + Behavior::Handles(operations) => operations, + }; + if !operations.iter().any(|name| name == operation) { + return json!({ + "ok": false, + "error": format!("invalid netd request: unknown variant `{operation}`"), + }); + } + match operation { + "prepare_bridge" | "prepare_macvtap" => json!({ + "ok": true, + "tap": "dtdeadbeef00", + "queues": request["queues"].as_u64().unwrap_or(1).max(1), + }), + "remove" | "check" => json!({"ok": true, "tap": "dtdeadbeef00"}), + "remove_all" => json!({"ok": true, "removed": 0, "incomplete": false}), + "list" => { + let instance = request["instance_id"].as_str().unwrap_or_default(); + let held: Vec = interfaces + .iter() + .filter(|record| { + instance.is_empty() || record["instance_id"].as_str() == Some(instance) + }) + .cloned() + .collect(); + json!({"ok": true, "interfaces": held}) + } + "remove_interface" => json!({"ok": true, "tap": request["tap"]}), + _ => json!({"ok": true}), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -868,6 +1587,7 @@ mod tests { qemu_uid: 1000, filtered: true, queues: 0, + workdir: String::new(), }; let filter = NetworkFilterConfig { mode: crate::config::NetworkFilterMode::Libvirt, @@ -895,7 +1615,6 @@ mod tests { fn remove_protocol_keeps_identity_fields_flat() { let request = Request::Remove { identity: identity("instance", "vm", 2), - filtered: true, }; let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "remove"); @@ -914,6 +1633,7 @@ mod tests { qemu_uid: 1000, filtered: true, queues: 0, + workdir: String::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -922,50 +1642,6 @@ mod tests { assert!(value.get("identity").is_none()); } - /// `filtered` says which of two shapes was built, and both are reachable - /// on any node this build can produce. There is no released peer that omits - /// it -- netd does not exist before v0.6 -- so it is required rather than - /// defaulted, and a request that leaves it out is a bug, not an old client. - #[test] - fn removal_states_which_shape_it_is_undoing() { - let error = serde_json::from_value::(serde_json::json!({ - "operation": "remove", - "instance_id": "instance", - "vm_id": "vm", - "nic_index": 0, - })) - .unwrap_err(); - assert!(error.to_string().contains("filtered"), "{error}"); - - for filtered in [true, false] { - let decoded: Request = serde_json::from_value(serde_json::json!({ - "operation": "remove", - "instance_id": "instance", - "vm_id": "vm", - "nic_index": 0, - "filtered": filtered, - })) - .unwrap(); - let Request::Remove { - filtered: decoded, .. - } = decoded - else { - panic!("wrong variant"); - }; - assert_eq!(decoded, filtered); - } - } - - /// A binding outlives the interface it was bound to, and TAP names are a - /// deterministic hash of the VM identity, so the same name comes back. - /// Removing an interface therefore clears whatever binding is there, and - /// only insists when the caller is about to create a replacement. - #[test] - fn binding_cleanup_insists_only_when_a_replacement_follows() { - assert_eq!(binding_cleanup(true), BindingCleanup::Required); - assert_eq!(binding_cleanup(false), BindingCleanup::BestEffort); - } - #[test] fn queue_counts_normalize_to_at_least_one_and_stay_bounded() { assert_eq!(validate_queues(0).unwrap(), 1); @@ -983,6 +1659,7 @@ mod tests { qemu_uid: 1000, filtered: false, queues: 4, + workdir: String::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["queues"], 4); @@ -1015,6 +1692,7 @@ mod tests { qemu_uid: 1000, mode: "private".into(), queues: 0, + workdir: String::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_macvtap"); @@ -1032,6 +1710,7 @@ mod tests { qemu_uid: 1000, filtered: true, queues: 0, + workdir: String::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -1052,7 +1731,7 @@ mod tests { drop(client); let result = timeout( Duration::from_secs(1), - serve_connection(&NetdConfig::default(), &mut server), + serve_connection(&std::sync::Arc::new(NetdConfig::default()), &mut server), ) .await; assert!(result.is_ok(), "disconnected peer blocked the handler"); @@ -1075,7 +1754,7 @@ mod tests { .await .unwrap(); client.shutdown().await.unwrap(); - serve_connection(&NetdConfig::default(), &mut server) + serve_connection(&std::sync::Arc::new(NetdConfig::default()), &mut server) .await .unwrap(); @@ -1108,6 +1787,7 @@ mod tests { qemu_uid: 1000, filtered: false, queues: 4, + workdir: String::new(), }; let filtering = NetworkFilterConfig { mode: crate::config::NetworkFilterMode::Libvirt, @@ -1158,6 +1838,7 @@ mod tests { qemu_uid: 1000, mode: "bridge".into(), queues: 4, + workdir: String::new(), }; let error = match prepare_macvtap("test:///default", &request, &filtering) { Err(error) => error, @@ -1179,6 +1860,7 @@ mod tests { qemu_uid: 1000, filtered: true, queues: 1, + workdir: String::new(), }; // Nothing on the wire can name a filter: the field does not exist. let wire = serde_json::to_value(Request::PrepareBridge(request.clone())).unwrap(); @@ -1208,4 +1890,306 @@ mod tests { assert!(error.to_string().contains("timed out")); assert!(started.elapsed() < Duration::from_secs(2)); } + + #[test] + fn the_workdir_travels_but_older_callers_may_omit_it() { + let request = Request::PrepareBridge(PrepareBridgeRequest { + identity: identity("instance", "vm", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: true, + queues: 1, + workdir: "/opt/dstack/run/vm/vm".into(), + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["workdir"], "/opt/dstack/run/vm/vm"); + // It is a log line, not an input, so a caller that never sets it is not + // asking for anything different. + let Request::PrepareBridge(decoded) = decode_minimal_bridge() else { + panic!("expected a bridge prepare"); + }; + assert_eq!(decoded.workdir, ""); + } + + /// A prepare carrying only the fields that predate this change. + fn decode_minimal_bridge() -> Request { + serde_json::from_value(serde_json::json!({ + "operation": "prepare_bridge", + "instance_id": "instance", + "vm_id": "vm", + "nic_index": 0, + "bridge": "br0", + "mac": "02:00:00:00:00:01", + "qemu_uid": 1000, + "filtered": true, + "queues": 1, + })) + .unwrap() + } + + #[test] + fn a_whole_vm_sweep_needs_no_record_of_what_it_is_deleting() { + let value = serde_json::to_value(Request::RemoveAll { + instance_id: "instance".into(), + vm_id: "vm".into(), + }) + .unwrap(); + assert_eq!(value["operation"], "remove_all"); + assert_eq!(value["instance_id"], "instance"); + assert_eq!(value["vm_id"], "vm"); + // No NIC index: the point is reaching the ones the caller can no longer + // name, so it names none and netd derives the whole space instead. + assert!(value.get("nic_index").is_none()); + + // That space is bounded by what an identity may say, which is what + // makes deriving it cheap enough to do on every launch. + let mut identity = identity("instance", "vm", MAX_NIC_INDEX); + assert!(validate_identity(&identity).is_ok()); + identity.nic_index = MAX_NIC_INDEX + 1; + assert!(validate_identity(&identity).is_err()); + } + + /// A sweep names no single interface, so its answer must not carry an + /// empty one. `request()` reads a blank `tap` as a malformed response, and + /// a third-party netd copying this shape would have to send a field that + /// means nothing. + #[test] + fn a_sweep_reports_a_count_and_no_interface() { + let value = serde_json::to_value(Outcome::Swept { removed: 3 }.into_response()).unwrap(); + assert_eq!(value["removed"], 3); + assert!(value.get("tap").is_none()); + + // A prepare still names one, because the caller hands that name to QEMU. + let value = serde_json::to_value(Outcome::tap("dtabc".into()).into_response()).unwrap(); + assert_eq!(value["tap"], "dtabc"); + assert!(value.get("removed").is_none()); + } + + /// The record is a hint; the name is the proof. Everything that can go + /// wrong with reading a string off an interface -- forged, truncated, + /// ambiguous, absent -- has to land in the same bucket, and it has to be + /// the bucket a collection treats conservatively. + #[test] + fn an_interface_says_whose_it_is_and_the_name_is_what_proves_it() { + let nic = identity("path-abc", "vm-1", 3); + let tap = tap_name(&nic); + let alias = interface_alias(&nic); + assert_eq!(alias, "dstack1:3:path-abc:vm-1"); + + let owner = owner_of(&tap, &alias).expect("its own record checks out"); + assert_eq!(owner.instance_id, "path-abc"); + assert_eq!(owner.vm_id, "vm-1"); + assert_eq!(owner.nic_index, 3); + + // A record naming some other interface proves nothing about this one. + // This is what makes the record unforgeable without making it + // authoritative: anything that can reach the socket can write a + // string, but only the true identity re-derives the name. + let forged = interface_alias(&identity("path-abc", "someone-elses-vm", 3)); + assert!(owner_of(&tap, &forged).is_none()); + assert!(owner_of(&tap, "").is_none()); + assert!(owner_of(&tap, "dstack1:3:path-abc").is_none()); + assert!(owner_of(&tap, &alias[..alias.len() - 2]).is_none()); + // A format this build does not know is not this format. + assert!(owner_of(&tap, &alias.replace("dstack1", "dstack2")).is_none()); + + // The two free-form fields are last and only the first of them has to + // be unambiguous, so a VM ID carrying the separator still reads back. + let odd = identity("path-abc", "vm:with:colons", 0); + assert_eq!( + owner_of(&tap_name(&odd), &interface_alias(&odd)).map(|owner| owner.vm_id), + Some("vm:with:colons".to_string()) + ); + // An instance ID carrying it is refused instead of mis-parsed. + assert!(validate_identity(&identity("path:abc", "vm-1", 0)).is_err()); + } + + /// An identity that cannot be recorded would produce an interface nothing + /// can attribute, which is the state this whole path exists to stop + /// creating. Refusing it is the only answer that keeps the invariant. + #[test] + fn an_identity_too_long_to_record_is_refused() { + let long = "v".repeat(128); + assert!(validate_identity(&identity("instance", &long, 0)).is_ok()); + let identity_too_long = identity(&"i".repeat(128), &long, 255); + assert!(interface_alias(&identity_too_long).len() > MAX_IFALIAS); + let error = validate_identity(&identity_too_long) + .unwrap_err() + .to_string(); + assert!(error.contains("too long to record"), "{error}"); + } + + /// The name space netd claims. A collection deletes what matches, so what + /// matches has to be exactly what netd can produce. + #[test] + fn the_managed_name_space_is_exactly_what_netd_produces() { + assert!(is_managed_name(&tap_name(&identity("instance", "vm", 0)))); + assert!(is_managed_name("dt0123456789ab")); + assert!(!is_managed_name("dt0123456789AB"), "digests are lower case"); + assert!(!is_managed_name("dt0123456789a"), "one short"); + assert!(!is_managed_name("dt0123456789abc"), "one long"); + assert!(!is_managed_name("dtzzzzzzzzzzzz")); + assert!(!is_managed_name("virbr0")); + assert!(!is_managed_name("eth0")); + // The whole space fits in IFNAMSIZ, or the kernel would refuse the + // names this reserves. + assert!(tap_name(&identity("instance", "vm", 255)).len() < 16); + } + + /// The command prints a table for a human and accepts no options to make it + /// print anything else, so this parses one. Narrowing to netd's own name + /// space is what makes that safe. + #[test] + fn the_binding_listing_reads_a_table_meant_for_a_person() { + let output = "\ + Port Dev Filter +--------------------------------- + dt1e053266e9f7 clean-traffic + dt28b105b3031a clean-traffic + vnet3 some-other-filter +"; + let names: HashSet = output + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|name| is_managed_name(name)) + .map(str::to_string) + .collect(); + assert_eq!(names.len(), 2); + assert!(names.contains("dt1e053266e9f7")); + // The header, the rule, and a binding that is not netd's all fall out. + assert!(!names.contains("Port")); + assert!(!names.contains("vnet3")); + } + + /// Everything else here reasons about strings. This puts the reasoning + /// next to the kernel: that an alias survives on a device netd actually + /// creates, that enumeration finds it, that the guards refuse what they + /// are meant to, and that removal leaves nothing. + /// + /// Refuses to run in the host's network namespace, so it cannot touch a + /// real node's interfaces even when it fails. Unsharing one from inside + /// the test is not enough: `/sys/class/net` keeps showing the old + /// namespace until sysfs is remounted, which is most of what `ip netns + /// exec` does. So it asks to be put in one: + /// + /// ```text + /// cargo test -p dstack-vmm --bins --no-run + /// sudo ip netns add dstack-netd-test + /// sudo ip netns exec dstack-netd-test \ + /// target/debug/deps/dstack_vmm- --ignored --test-threads=1 + /// sudo ip netns del dstack-netd-test + /// ``` + #[test] + #[ignore = "needs root and its own network namespace; see the doc comment"] + fn a_real_interface_carries_its_record_and_removal_leaves_nothing() { + assert!( + nix::unistd::Uid::effective().is_root(), + "this test needs root" + ); + let (mine, init) = ( + std::fs::read_link("/proc/self/ns/net").unwrap(), + std::fs::read_link("/proc/1/ns/net").unwrap(), + ); + assert_ne!( + mine, init, + "run this inside its own network namespace; it creates and deletes interfaces" + ); + // Nothing in this namespace to talk to, which is also the state of a + // node that does not filter: the listing has to work without libvirt. + let uri = "qemu:///nonexistent-for-this-test"; + + let nic = identity("test-instance", "vm-1", 2); + let tap = tap_name(&nic); + ip(&["tuntap", "add", "dev", &tap, "mode", "tap"]).unwrap(); + set_alias(&tap, &nic).unwrap(); + assert!(is_tuntap(&tap), "the kernel publishes tun_flags for a TAP"); + + let records = list_interfaces(uri, ""); + let record = records + .iter() + .find(|record| record.tap == tap) + .expect("an interface netd created is one netd can find"); + assert_eq!(record.instance_id.as_deref(), Some("test-instance")); + assert_eq!(record.vm_id.as_deref(), Some("vm-1")); + assert_eq!(record.nic_index, Some(2)); + assert_eq!(record.kind, "tap"); + // Narrowing by instance is what keeps one VMM's collection off + // another's interfaces. + assert_eq!(list_interfaces(uri, "test-instance").len(), 1); + assert!(list_interfaces(uri, "someone-else").is_empty()); + + // A device with one of netd's names that netd did not create. The name + // is 48 bits of digest, so this is not about collisions -- it is that + // `ip link delete` does not ask what it is deleting, and netd runs as + // root. + let impostor = tap_name(&identity("test-instance", "not-a-tap", 0)); + ip(&["link", "add", &impostor, "type", "dummy"]).unwrap(); + assert!(is_managed_name(&impostor)); + assert!( + !list_interfaces(uri, "") + .iter() + .any(|record| record.tap == impostor), + "a device netd did not create is not offered up for collection" + ); + let refused = remove_interface(uri, &impostor, BindingCleanup::Skip).unwrap_err(); + assert!(refused.to_string().contains("refusing to delete")); + + // An interface whose record does not re-derive its own name proves + // nothing, and lands in the same bucket as no record at all. + ip(&[ + "link", + "set", + "dev", + &tap, + "alias", + "dstack1:2:test-instance:some-other-vm", + ]) + .unwrap(); + let records = list_interfaces(uri, ""); + let record = records.iter().find(|record| record.tap == tap).unwrap(); + assert!(record.instance_id.is_none(), "a forged record is no record"); + + remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); + assert!(!Path::new("/sys/class/net").join(&tap).exists()); + assert!(!list_interfaces(uri, "") + .iter() + .any(|record| record.tap == tap)); + // Removing what is not there is not an error: a sweep derives names + // and most of them miss. + remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); + } + /// A netd that answers a sweep with no count did not sweep. Reading the + /// absent field as zero is the same conflation `queues` is shaped to + /// avoid, and here it would report a netd that cannot collect a VM's + /// interfaces as a VM that had none. + #[tokio::test] + async fn a_sweep_without_a_count_is_not_read_as_an_empty_one() { + // Claims the operation, answers without the field. + let netd = testing::FakeNetd::spawn(testing::Behavior::handling(&[])); + let error = remove_all(netd.socket(), "instance", "vm") + .await + .expect_err("an answer with no count is not a successful sweep"); + assert!(!is_unreachable(&error)); + + let netd = testing::FakeNetd::spawn(testing::Behavior::handling(&["remove_all"])); + let removed = remove_all(netd.socket(), "instance", "vm").await.unwrap(); + assert_eq!(removed, 0); + } + + /// Every "an unreachable netd is not a failure" branch in the VMM hangs + /// off this one predicate, and a marker attached as context is not a link + /// in the source chain. + #[test] + fn an_unreachable_netd_is_recognized_through_the_contexts_stacked_on_it() { + let error = anyhow::Error::from(std::io::Error::from(std::io::ErrorKind::NotFound)) + .context(Unreachable) + .context("failed to connect to netd at /run/netd.sock") + .context("failed to prepare netd-managed networking"); + assert!(is_unreachable(&error)); + + let other = anyhow::anyhow!("netd remove_all failed: no such bridge") + .context("failed to prepare netd-managed networking"); + assert!(!is_unreachable(&other)); + } } diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index d7500b008..62c4bd78b 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -3,9 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 use crate::app::{ - clamp_queues_without_netd, make_sys_config, needs_netd_interface, resolved_networks, - settle_vhost, simulator_config_for_manifest, sync_tee_simulator_config, Image, VmConfig, - VmWorkDir, + make_sys_config, needs_netd_interface, resolved_networks, settle_vhost, + simulator_config_for_manifest, sync_tee_simulator_config, Image, VmConfig, VmWorkDir, }; use crate::config::Config; use crate::main_service; @@ -280,42 +279,19 @@ Compose file content (first 200 chars): gateway_enabled: app_compose.gateway_enabled(), }; - // One-shot has no netd lifecycle, so a bridge NIC that only wanted the - // vCPU-scaled default drops to a single queue here exactly as it would on a - // server without netd. Anything still needing an interface was asked for - // explicitly, and is refused rather than silently downgraded. - let requested = if manifest.networks.is_empty() { - vec![config.cvm.networking.nic.clone()] - } else { - manifest.networks.clone() - }; let mut runtime_networks = resolved_networks(&manifest, &config.cvm); - let clamped = clamp_queues_without_netd(&requested, &mut runtime_networks, &config.cvm, false); - // The server settles vhost after clamping, because clamping changes whether - // a NIC needs netd and that changes which netdev it gets. Skipping it here - // left `vhost_enabled()` reading as a request rather than a decision, so - // the launch warned about a `/dev/vhost-net` the netdev it then built does - // not open. - let vhost_denied = settle_vhost(&mut runtime_networks, &config.cvm); - if vhost_denied > 0 { - tracing::warn!( - "no qemu-bridge-helper found, so {vhost_denied} bridge interface(s) fall back to the \ - non-vhost bridge netdev; set cvm.qemu_bridge_helper to enable vhost" - ); - } - if clamped > 0 { - tracing::warn!( - "one-shot execution has no netd, so {clamped} bridge interface(s) fall back to a \ - single queue pair; run the VMM server to let queue pairs scale with vCPUs" - ); - } - if !dry_run - && runtime_networks - .iter() - .any(|network| needs_netd_interface(network, &config.cvm)) - { + // Settle the data plane before anything reads it, so `vhost_enabled()` is a + // decision rather than a request and the launch does not warn about a + // `/dev/vhost-net` the netdev it then builds never opens. + settle_vhost(&mut runtime_networks); + // Bridge and macvtap host interfaces belong to netd, whose lifecycle + // one-shot does not manage. Refusing is the honest answer: the alternative + // was to quietly build a different interface here than the server would, + // and then report the VM as if it had the one it asked for. + if !dry_run && runtime_networks.iter().any(needs_netd_interface) { anyhow::bail!( - "one-shot execution does not manage netd interface lifecycle; run the VMM server directly or use --dry-run" + "one-shot execution does not manage netd interface lifecycle, which bridge and \ + macvtap networking need; run the VMM server directly or use --dry-run" ); } diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 20e515520..f4e1a7b52 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -321,17 +321,30 @@ def encrypt_env(envs, hex_public_key: str) -> str: def parse_port_mapping(port_str: str) -> Dict: - """Parse a port mapping string into a dictionary.""" + """Parse a port mapping string into a dictionary. + + Accepts an optional "@" suffix naming which NIC the traffic enters + through. Without it the VMM picks: the first user-mode NIC, else the first + bridge NIC. A single-NIC VM never needs it. + """ + nic_index = None + if "@" in port_str: + port_str, _, nic = port_str.rpartition("@") + # `int()` alone would take "1_0" as 10, " 1" as 1, and "+1" as 1. A NIC + # index is a position in a list the user wrote, so only digits are it. + if not (nic.isascii() and nic.isdigit()): + raise argparse.ArgumentTypeError(f"Invalid NIC index: {nic}") + nic_index = int(nic) parts = port_str.split(":") if len(parts) == 3: - return { + mapping = { "protocol": parts[0], "host_address": "127.0.0.1", "host_port": int(parts[1]), "vm_port": int(parts[2]), } elif len(parts) == 4: - return { + mapping = { "protocol": parts[0], "host_address": parts[1], "host_port": int(parts[2]), @@ -339,6 +352,9 @@ def parse_port_mapping(port_str: str) -> Dict: } else: raise argparse.ArgumentTypeError(f"Invalid port mapping format: {port_str}") + if nic_index is not None: + mapping["nic_index"] = nic_index + return mapping def read_utf8(filepath: str) -> str: @@ -1907,7 +1923,7 @@ def _patched_format_help(): "--port", action="append", type=str, - help="Port mapping in format: protocol[:address]:from:to", + help="Port mapping in format: protocol[:address]:from:to[@nic]", ) deploy_parser.add_argument( "--gpu", @@ -2063,7 +2079,7 @@ def _patched_format_help(): action="append", type=str, required=True, - help="Port mapping in format: protocol[:address]:from:to (can be used multiple times)", + help="Port mapping in format: protocol[:address]:from:to[@nic] (can be used multiple times)", ) # Update (all-in-one) command @@ -2133,7 +2149,7 @@ def _patched_format_help(): "--port", action="append", type=str, - help="Port mapping in format: protocol[:address]:from:to (can be used multiple times)", + help="Port mapping in format: protocol[:address]:from:to[@nic] (can be used multiple times)", ) port_group.add_argument( "--no-ports", diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index 27adcaecd..b07b36255 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -293,7 +293,10 @@ const CreateVmDialogComponent = {
- +
diff --git a/dstack/vmm/ui/src/components/PortMappingEditor.ts b/dstack/vmm/ui/src/components/PortMappingEditor.ts index 0668bd952..70a4ad278 100644 --- a/dstack/vmm/ui/src/components/PortMappingEditor.ts +++ b/dstack/vmm/ui/src/components/PortMappingEditor.ts @@ -8,6 +8,10 @@ type PortEntry = { host_port: number | null; vm_port: number | null; custom_ip?: string; // User-entered IP for custom mode + // Which NIC the traffic enters through. Blank lets the VMM pick: the first + // user-mode NIC, else the first bridge NIC. A single-NIC VM never needs it, + // which is why the field only appears once a VM has more than one. + nic_index?: number | string | null; }; // ... keep your types as-is ... @@ -21,6 +25,9 @@ const PortMappingEditorComponent = { name: 'PortMappingEditor', props: { ports: { type: Array, required: true }, + // How many NICs the VM has. One NIC has nothing to choose between, so the + // column stays hidden rather than offering a pin that can only be 0. + nicCount: { type: Number, default: 1 }, }, // normalize on initial load @@ -63,6 +70,15 @@ const PortMappingEditorComponent = { + @@ -95,6 +111,7 @@ const PortMappingEditorComponent = { custom_ip: '', host_port: null, vm_port: null, + nic_index: null, }); }, diff --git a/dstack/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts index ec2bfbfa6..4fa40e493 100644 --- a/dstack/vmm/ui/src/components/UpdateVmDialog.ts +++ b/dstack/vmm/ui/src/components/UpdateVmDialog.ts @@ -176,7 +176,10 @@ const UpdateVmDialogComponent = {
- +
diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index 70e9db9df..12bbbf6de 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -107,6 +107,15 @@ type PortFormEntry = { host_address?: string; host_port?: number | null; vm_port?: number | null; + /** + * Which NIC this mapping's traffic enters through. Unset lets the VMM pick. + * Carried through edits unchanged: `GetInfo` reports it and this form sends + * the whole list back, so dropping it here would silently unpin a mapping + * whenever anyone touched an unrelated field. + */ + // `v-model.number` leaves the raw string here when it does not parse, so + // an emptied box is `''` rather than `null`. See `normalizePorts`. + nic_index?: number | string | null; }; type NetworkFormEntry = { @@ -435,6 +444,7 @@ fi host_address: port.host_address || '127.0.0.1', host_port: typeof port.host_port === 'number' ? port.host_port : null, vm_port: typeof port.vm_port === 'number' ? port.vm_port : null, + nic_index: typeof port.nic_index === 'number' ? port.nic_index : null, })); const normalizePorts = (ports: PortFormEntry[] = []): VmmTypes.IPortMapping[] => @@ -445,11 +455,23 @@ fi port.host_port === null || port.host_port === undefined ? Number.NaN : Number(port.host_port); const vmPort = port.vm_port === null || port.vm_port === undefined ? Number.NaN : Number(port.vm_port); + // An unpinned mapping must stay unpinned rather than become NIC 0: + // the VMM's own default is the first user-mode NIC, not the first NIC. + // `v-model.number` hands back the raw string when it does not parse, + // so a box the operator cleared arrives as `''`. `Number('')` is 0, + // which would pin to NIC 0 the mapping they just unpinned. + const nicIndex = + port.nic_index === null || port.nic_index === undefined || port.nic_index === '' + ? undefined + : Number(port.nic_index); return { protocol, host_address: (port.host_address || '127.0.0.1').trim() || '127.0.0.1', host_port: hostPort, vm_port: vmPort, + ...(Number.isInteger(nicIndex) && (nicIndex as number) >= 0 + ? { nic_index: nicIndex } + : {}), }; }) .filter( @@ -457,13 +479,7 @@ fi port.protocol.length > 0 && Number.isFinite(port.host_port) && Number.isFinite(port.vm_port), - ) - .map((port) => ({ - protocol: port.protocol, - host_address: port.host_address, - host_port: port.host_port, - vm_port: port.vm_port, - })); + ); const cloneNetworks = (configuration?: VmConfiguration | null): NetworkFormEntry[] => { const configured = configuration?.networks && configuration.networks.length > 0 diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 3a83d5016..9ef0ecec4 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -46,7 +46,11 @@ max_allocable_memory_in_mb = 100_000 # MB qmp_socket = false # The user to run the VM as. If empty, the VM will be run as the current user. # Unique namespace when multiple dstack-vmm instances share one host. When -# empty, a stable value is derived from run_path. +# empty, a stable value is derived from run_path. It is what netd records on +# every host interface this VMM asks for, and the name space those interfaces' +# names are derived in. Two VMMs on one host must not share one: each would +# build interfaces at names the other can also produce. Left empty it is +# derived from run_path, which cannot collide. May not contain ":". instance_id = "" # Network choices that deployment RPC callers may make. Macvtap is excluded # by default because libvirt nwfilter applies only to bridge interfaces. @@ -58,11 +62,7 @@ allowed_macvtap_parents = [] # vhost on, queue pairs otherwise default to the VM's vCPU count, capped at # 16 (without vhost the default is a single queue pair); raising this # above 16 widens what a caller may request without moving that default, and -# lowering it below 16 lowers the default too. Bridge mode needs netd for -# anything above 1, because qemu-bridge-helper cannot create a multiqueue TAP. -# Without netd an unfiltered bridge NIC that took the default drops to one -# queue; one that asked for a count keeps it and fails to launch instead, so -# the caller learns their request was not met. +# lowering it below 16 lowers the default too. max_net_queues = 16 use_mrconfigid = true @@ -72,10 +72,6 @@ use_mrconfigid = true #qemu_version = "" qemu_pci_hole64_size = 0 qemu_hotplug_off = false -# Path to qemu-bridge-helper, needed by vhost bridge networking because QEMU's -# `tap` netdev, unlike its `bridge` netdev, has no compiled-in default. Empty -# probes the known distribution locations. -#qemu_bridge_helper = "/usr/lib/qemu/qemu-bridge-helper" # TDX attestation/hash scheme policy: # - "legacy": digest.txt + legacy verifier # - "lite": digest.txt + measurement.tdx.cbor + no-QEMU verifier @@ -144,17 +140,18 @@ restrict = false # bridge = "virbr0" # Optional filtering for bridge interfaces only. It does not apply to macvtap. -# "none" installs no nwfilter binding. It does not by itself remove the netd -# dependency: netd also builds the multiqueue TAP that qemu-bridge-helper -# cannot create, so a bridge node without netd is limited to one queue pair. +# "none" installs no nwfilter binding. It does not remove the netd dependency: +# netd builds every bridge and macvtap host interface whatever this says. [cvm.network_filter] mode = "none" filter = "clean-traffic" parameters = {} -# Shared privileged networking service. Used for macvtap NICs, for libvirt -# filtering, and for multiqueue bridge NICs. Socket filesystem permissions -# authorize clients. +# Shared privileged networking service. Required by bridge and macvtap +# networking: it builds every host interface those modes use, binds their +# nwfilters, and releases them again. It does not forward host ports. User mode +# and a caller-supplied netdev need nothing from it. Socket filesystem +# permissions authorize clients. [netd] socket = "/run/dstack/netd.sock" # Applied when netd creates the socket itself. A systemd socket unit controls From 2a3f182f6adb960482a2384e02117138cc1dad27 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 20:14:18 -0700 Subject: [PATCH 4/5] fix(vmm): keep interrupted removals recoverable --- docs/bridge-networking.md | 11 +++--- docs/vmm-cli-user-guide.md | 4 +- dstack/vmm/src/app.rs | 79 +++++++++++++++++++++++++++++--------- 3 files changed, 69 insertions(+), 25 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 43216d086..df961c2bb 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -210,8 +210,9 @@ vmm-cli.py deploy ... --port udp:0.0.0.0:7483:51820@0 --port tcp:127.0.0.1:7484: ``` Leave `@` off and the VMM picks the first user-mode NIC — where QEMU's -`hostfwd=` entries have always gone — and failing that the first bridge NIC. A -single-NIC VM never needs it. +`hostfwd=` entries have always gone. If the VM has no user-mode NIC, the mapping +has no publishing backend and the launch log names it as stranded. A single-NIC +user-mode VM never needs the suffix. With several NICs the choice used to be made silently, and not always the way an operator would have. A bridge NIC for external traffic beside a user-mode NIC for @@ -221,9 +222,9 @@ whatever the bridge NIC's nwfilter was there to enforce and hiding the client's address behind the slirp gateway. A second user-mode NIC could never publish anything at all, because only the first was ever selected. -A mapping resolves to exactly one NIC, and that NIC's backend decides the -mechanism: `hostfwd=` for user mode, `netd` for a bridge. Nothing can be claimed -by both. +A mapping resolves to at most one NIC. The only backend that can carry it is +QEMU user networking through `hostfwd=`; `netd` builds bridge interfaces but +does not publish host ports. ### Which ports a bridge NIC can publish diff --git a/docs/vmm-cli-user-guide.md b/docs/vmm-cli-user-guide.md index 1f803e31c..6dddc2ba5 100644 --- a/docs/vmm-cli-user-guide.md +++ b/docs/vmm-cli-user-guide.md @@ -292,8 +292,8 @@ Expose services running in your VM: --port tcp:8080:80 --port tcp:8443:443 # Pin a mapping to one NIC: protocol[:host_address]:host_port:vm_port@ -# Without @ the mapping enters through the first user-mode NIC, or the -# first bridge NIC when the VM has no user-mode one. +# Without @ the mapping enters through the first user-mode NIC. A VM +# with no user-mode NIC has no backend that can publish the mapping. --port tcp:0.0.0.0:8443:443@0 ``` diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index f6deaa8ec..1d983dc0a 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -531,7 +531,7 @@ impl App { .await .context("GPU sanitization task failed")??; if let Err(error) = self - .prepare_filtered_networks(&vm_config, &mut runtime_networks) + .prepare_netd_networks(&vm_config, &mut runtime_networks) .await { let _ = work_dir.clear_runtime_networks(); @@ -612,7 +612,7 @@ impl App { Ok(()) } - async fn prepare_filtered_networks( + async fn prepare_netd_networks( &self, vm: &VmConfig, networks: &mut [Networking], @@ -931,15 +931,25 @@ impl App { } } + // Clear the in-memory mark if anything below fails before ownership of + // the removal is handed to the background task. In particular, a + // removal that cannot persist its crash-recovery marker must not start: + // without that marker a netd outage could leave interfaces no later + // VMM start knows to release. + let mut mark = RemovalMark::new(self.clone(), id); + // Persist the removing marker so crash recovery can resume let work_dir = self.work_dir(id)?; - if let Err(err) = work_dir.set_removing() { - warn!("failed to write .removing marker for {id}: {err:?}"); - } + work_dir + .set_removing() + .with_context(|| format!("failed to write .removing marker for {id}"))?; // User-initiated removal always deletes the workdir let app = self.clone(); let id = id.to_string(); + // `finish_remove_vm` owns the mark from here. Do not clear it in the + // request task while the background removal is still running. + mark.retain(); tokio::spawn(async move { if let Err(err) = app.finish_remove_vm(&id, true).await { error!("Background cleanup failed for {id}: {err:?}"); @@ -954,12 +964,10 @@ impl App { /// /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup. async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> { - // Every exit from here clears the mark, including the `?`s below and a - // panic in this task, which `tokio::spawn` would otherwise swallow. - let _mark = RemovalMark { - app: self.clone(), - id: id.to_string(), - }; + // Ordinary exits clear the mark, including the `?`s below and a panic + // in this task, which `tokio::spawn` would otherwise swallow. The one + // recoverable incomplete outcome explicitly retains it. + let mut mark = RemovalMark::new(self.clone(), id); // Held across the stop, the wait and the release, not just the release. // `removing` turns launches away, but a launch that passed that check // before the marker was set is already inside the lock: it has not @@ -1036,6 +1044,10 @@ impl App { "VM {id} keeps its directory because netd did not release its interfaces; \ the removal resumes at the next VMM start" ); + // The disk marker says this removal must be retried. Keep the + // matching in-memory state too, so start/update/stop cannot revive + // the VM before that retry happens. + mark.retain(); return Ok(()); } else if vm_path.path().exists() { if let Err(err) = fs::remove_dir_all(&vm_path) { @@ -2253,17 +2265,13 @@ mod tests { assert!(!app.lock().start_removing("orphan")); } - /// `finish_remove_vm` returns early on more than its happy path. A mark it - /// left behind is not a stale flag: every later operation on that VM, - /// including the removal that would retry, answers "being removed". + /// An ordinary early error clears the in-memory mark so the caller can + /// retry rather than leaving a VM permanently inaccessible. #[tokio::test] async fn a_removal_that_gives_up_early_does_not_leave_the_vm_marked() { let app = test_app(); { - let _mark = RemovalMark { - app: app.clone(), - id: "vm-1".to_string(), - }; + let _mark = RemovalMark::new(app.clone(), "vm-1"); assert!(app.lock().start_removing("vm-1")); assert!(app.refuse_if_removing("vm-1").is_err()); } @@ -2273,6 +2281,22 @@ mod tests { ); } + /// Once a removal deliberately keeps its crash-recovery marker, dropping + /// the guard must not make the VM launchable again in the current process. + #[tokio::test] + async fn a_removal_waiting_for_netd_stays_marked() { + let app = test_app(); + assert!(app.lock().start_removing("vm-1")); + { + let mut mark = RemovalMark::new(app.clone(), "vm-1"); + mark.retain(); + } + assert!( + app.refuse_if_removing("vm-1").is_err(), + "the in-memory state must agree with the retained disk marker" + ); + } + /// A restart decided before a stop must not outlive it. The restart task /// reads the started flag off disk and only then queues a launch, which /// waits for the lock the stop is holding; without a re-read under that @@ -3482,10 +3506,29 @@ impl AppState { struct RemovalMark { app: App, id: String, + clear_on_drop: bool, +} + +impl RemovalMark { + fn new(app: App, id: &str) -> Self { + Self { + app, + id: id.to_string(), + clear_on_drop: true, + } + } + + /// Leave the removal mark in place after this guard goes out of scope. + fn retain(&mut self) { + self.clear_on_drop = false; + } } impl Drop for RemovalMark { fn drop(&mut self) { + if !self.clear_on_drop { + return; + } let mut state = self.app.lock(); state.removing.remove(&self.id); if let Some(vm) = state.vms.get_mut(&self.id) { From 51a5ae130072d69e87143b9c4e799483ce3c81f0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 7 Sep 2026 23:57:29 -0700 Subject: [PATCH 5/5] fix(vmm): preserve network cleanup across failures and topology changes --- docs/bridge-networking.md | 11 ++ dstack/vmm/src/app.rs | 182 ++++++++++++++++++++++++++++++--- dstack/vmm/src/app/workdir.rs | 20 ++++ dstack/vmm/src/main_service.rs | 3 +- dstack/vmm/src/netd.rs | 87 ++++++++++++++-- 5 files changed, 280 insertions(+), 23 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index df961c2bb..f627851dc 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -283,6 +283,17 @@ directory stays and the next VMM start resumes the removal; `remove_all` is idempotent, so the retry costs one round trip. A VM that never asked `netd` for an interface is unaffected: there is nothing for `netd` to be holding. +The VMM persists `.netd-pending` before asking netd to prepare an interface and +clears it only after a successful whole-VM sweep. This cleanup marker survives +failed launches and network configuration changes, even if the runtime snapshot +is absent or replaced by a user-mode topology. Older snapshots are promoted to +the marker before cleanup or replacement. A failed cleanup during an update +also leaves the old snapshot intact. + +On an unfiltered node, an unavailable `libvirtd` does not make an otherwise +successful TAP sweep fail. Filtered nodes still require confirmation that their +nwfilter bindings have been released; deleting the TAP alone is not sufficient. + What no VMM will retry is an interface whose VM directory an operator deleted by hand, or one recorded under an instance ID no VMM uses any more. `netd list` shows both, with the instance and VM they are recorded under: diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 1d983dc0a..518bae52d 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -530,13 +530,10 @@ impl App { }) .await .context("GPU sanitization task failed")??; - if let Err(error) = self - .prepare_netd_networks(&vm_config, &mut runtime_networks) - .await - { - let _ = work_dir.clear_runtime_networks(); - return Err(error); - } + // Keep any earlier snapshot on failure: preparation may not have + // been able to persist the cleanup marker for legacy NICs yet. + self.prepare_netd_networks(&vm_config, &mut runtime_networks) + .await?; let processes = match vm_config.config_qemu( &work_dir, &self.config.cvm, @@ -560,12 +557,13 @@ impl App { } for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { - self.release_vm_interfaces(&vm_config.manifest.id).await; - if let Err(clear_err) = work_dir.clear_runtime_networks() { - warn!( - id, - "failed to clear runtime networks after start failure: {clear_err}" - ); + if self.release_vm_interfaces(&vm_config.manifest.id).await { + if let Err(clear_err) = work_dir.clear_runtime_networks() { + warn!( + id, + "failed to clear runtime networks after start failure: {clear_err}" + ); + } } if let Some(vm_state) = self.lock().get_mut(id) { vm_state.state.runtime_networks.clear(); @@ -643,10 +641,21 @@ impl App { // it no longer wants an interface, and gating it on wanting one is how // the interfaces it left behind would become unreachable to every // later launch. + let vm_workdir = self.work_dir(&vm.manifest.id)?; + if vm_workdir + .runtime_networks() + .iter() + .any(needs_netd_interface) + { + // Fail before a user-mode boot can overwrite a legacy snapshot + // when its replacement cleanup marker cannot be persisted. + vm_workdir.mark_network_cleanup_pending()?; + } self.release_vm_interfaces(&vm.manifest.id).await; if !networks.iter().any(needs_netd_interface) { return Ok(()); } + vm_workdir.mark_network_cleanup_pending()?; let qemu_uid = Uid::effective().as_raw(); // Only ever read back out of a log line: netd is told where the VM // lives so an operator holding an opaque TAP name can reach the VM @@ -854,6 +863,22 @@ impl App { /// directory is what says to try again, and deleting it over a failed /// release is what strands an interface with nothing left to reach it. pub(crate) async fn release_vm_interfaces(&self, vm_id: &str) -> bool { + let workdir = match self.work_dir(vm_id) { + Ok(workdir) => workdir, + Err(error) => { + warn!(vm_id, %error, "cannot locate network cleanup state"); + return false; + } + }; + // Upgrade snapshots written before the marker existed. A failed sweep + // must remain recoverable even if the next boot replaces the snapshot + // with user-mode NICs or an update clears it. + if workdir.runtime_networks().iter().any(needs_netd_interface) { + if let Err(error) = workdir.mark_network_cleanup_pending() { + warn!(vm_id, %error, "cannot persist pending network cleanup"); + return false; + } + } // Ask for the release, rather than asking whether it can be asked for. // A probe first would put a second round trip in front of every stop // and -- worse -- would make a *busy* netd look like an absent one and @@ -868,6 +893,10 @@ impl App { .await { Ok(removed) => { + if let Err(error) = workdir.clear_network_cleanup_pending() { + warn!(vm_id, %error, "cannot clear pending network cleanup"); + return false; + } if removed > 0 { info!(vm_id, removed, "released netd-managed interfaces"); } @@ -1023,7 +1052,8 @@ impl App { // A VM that never asked netd for an interface -- user mode, a custom // netdev, or one that never launched -- has nothing for netd to be // holding, so an absent netd is not a reason to keep its directory. - let held_interfaces = vm_path.runtime_networks().iter().any(needs_netd_interface); + let held_interfaces = vm_path.network_cleanup_pending() + || vm_path.runtime_networks().iter().any(needs_netd_interface); let released = self.release_vm_interfaces(id).await; // Only delete the workdir for user-initiated removal or if .removing marker exists. @@ -2180,6 +2210,130 @@ mod tests { ) } + /// A supervisor with no remaining process lets removal exercise the real + /// directory and netd paths without launching a VM. + async fn stopped_supervisor() -> (SupervisorClient, tokio::task::JoinHandle<()>) { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut reader = BufReader::new(&mut stream); + let mut header = String::new(); + loop { + header.clear(); + assert!(reader.read_line(&mut header).await.unwrap() > 0); + if header == "\r\n" { + break; + } + } + let body = serde_json::to_string(&supervisor_client::supervisor::Response::Data( + Option::::None, + )) + .unwrap(); + stream + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body, + ) + .as_bytes(), + ) + .await + .unwrap(); + } + }); + (SupervisorClient::new(&format!("http://{address}")), task) + } + + #[tokio::test] + async fn failed_release_survives_snapshot_replacement_and_removal_retry() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(&dir.path().join("absent-netd.sock"), dir.path()), + supervisor.clone(), + ); + let workdir = app.work_dir("vm-1").unwrap(); + std::fs::create_dir_all(workdir.path()).unwrap(); + // A legacy launch only left the runtime snapshot, without a marker. + let network = Networking { + nic: NicNetworking { + mode: NetworkingMode::Bridge, + ..Default::default() + }, + ..Default::default() + }; + workdir.set_runtime_networks(&[network]).unwrap(); + assert!(!workdir.network_cleanup_pending()); + assert!(!app.release_vm_interfaces("vm-1").await); + assert!(workdir.network_cleanup_pending()); + // A subsequent user-mode boot can replace the snapshot. Cleanup must + // still remember the bridge interfaces from the earlier boot. + workdir + .set_runtime_networks(&[Networking::default()]) + .unwrap(); + workdir.set_removing().unwrap(); + assert!(app.lock().start_removing("vm-1")); + app.finish_remove_vm("vm-1", true).await.unwrap(); + assert!(workdir.path().exists()); + assert!(workdir.is_removing()); + assert!(workdir.network_cleanup_pending()); + assert!(app.refuse_if_removing("vm-1").is_err()); + + // Simulate restarting the VMM after netd has recovered. + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); + let restarted = App::new(test_config(netd.socket(), dir.path()), supervisor); + restarted.finish_remove_vm("vm-1", true).await.unwrap(); + assert!(!workdir.path().exists()); + assert_eq!(netd.operations(), vec!["remove_all"]); + server.abort(); + } + + #[tokio::test] + async fn removal_without_a_snapshot_distinguishes_pending_from_never_prepared() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(&dir.path().join("absent-netd.sock"), dir.path()), + supervisor, + ); + for (id, pending) in [("interrupted-prepare", true), ("user-only", false)] { + let workdir = app.work_dir(id).unwrap(); + std::fs::create_dir_all(workdir.path()).unwrap(); + if pending { + // Preparation wrote this before contacting netd, then crashed + // without ever writing runtime-networks.json. + workdir.mark_network_cleanup_pending().unwrap(); + } + workdir.set_removing().unwrap(); + app.finish_remove_vm(id, true).await.unwrap(); + assert_eq!(workdir.path().exists(), pending); + assert_eq!(workdir.is_removing(), pending); + } + server.abort(); + } + + #[tokio::test] + async fn successful_sweep_clears_the_pending_marker() { + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(netd.socket(), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + let workdir = app.work_dir("vm-1").unwrap(); + std::fs::create_dir_all(workdir.path()).unwrap(); + workdir.mark_network_cleanup_pending().unwrap(); + assert!(app.release_vm_interfaces("vm-1").await); + assert!(!workdir.network_cleanup_pending()); + assert!(workdir.path().exists()); + } + /// A netd outage must not become a fleet that cannot be stopped. #[tokio::test] async fn a_stop_survives_a_netd_that_is_not_there() { diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index a9822c7ec..5809f73ba 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -143,6 +143,26 @@ impl VmWorkDir { self.workdir.join("runtime-networks.json") } + /// Written before asking netd to create anything, and removed only after + /// a successful whole-VM sweep. Unlike the runtime snapshot, this survives + /// failed launches and changes to a topology that no longer uses netd. + pub fn mark_network_cleanup_pending(&self) -> Result<()> { + safe_write::safe_write(self.workdir.join(".netd-pending"), b"") + .context("failed to persist pending network cleanup") + } + + pub fn network_cleanup_pending(&self) -> bool { + self.workdir.join(".netd-pending").exists() + } + + pub fn clear_network_cleanup_pending(&self) -> Result<()> { + match fs::remove_file(self.workdir.join(".netd-pending")) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).context("failed to clear pending network cleanup"), + } + } + pub fn runtime_networks(&self) -> Vec { fs::read_to_string(self.runtime_networks_path()) .ok() diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 1c2835942..ea4166dcc 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -1035,8 +1035,7 @@ impl VmmRpc for RpcHandler { .info(&request.id) .await? .is_some_and(|info| info.state.status.is_running()); - if !is_running { - self.app.release_vm_interfaces(&request.id).await; + if !is_running && self.app.release_vm_interfaces(&request.id).await { vm_work_dir.clear_runtime_networks()?; } manifest.networks = networks; diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index aca287a9c..52aeeb4ae 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -675,7 +675,12 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { Ok(Outcome::Listed(list_interfaces(libvirt_uri, &instance_id))) } Request::RemoveAll { instance_id, vm_id } => { - let removed = sweep_vm_interfaces(libvirt_uri, &instance_id, &vm_id)?; + let removed = sweep_vm_interfaces( + libvirt_uri, + &instance_id, + &vm_id, + config.filter_policy().requires_binding(), + )?; Ok(Outcome::Swept { removed }) } Request::RemoveInterface { tap } => { @@ -919,8 +924,8 @@ enum BindingCleanup { /// /// `validate_identity` caps the NIC index, so the whole space a VM can occupy /// is enumerable: 256 names, each a `stat` that usually misses. Cleanup is -/// best-effort about bindings -- nothing is about to take these names, and a -/// node running unfiltered TAPs need not have libvirtd at all. +/// best-effort about unknown bindings on unfiltered nodes, which need not have +/// libvirtd at all. Filtered nodes must confirm their bindings were released. /// /// A name with no interface is not skipped. An nwfilter binding outlives the /// TAP it was bound to, so the one state teardown must not leave behind is @@ -929,7 +934,12 @@ enum BindingCleanup { /// less than the thing it replaced is not a sweep. Those names are decided /// against a single listing, because the whole point of enumerating a bounded /// space is that deciding one name stays cheap. -fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Result { +fn sweep_vm_interfaces( + libvirt_uri: &str, + instance_id: &str, + vm_id: &str, + requires_binding: bool, +) -> Result { let identity = InterfaceIdentity { instance_id: instance_id.to_string(), vm_id: vm_id.to_string(), @@ -945,7 +955,8 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res // out by trying, once. let mut libvirt = true; let mut removed = 0; - let mut first_error = None; + let mut first_error = (requires_binding && bindings.is_none()) + .then(|| anyhow::anyhow!("cannot confirm nwfilter cleanup: binding listing failed")); for nic_index in 0..=MAX_NIC_INDEX { let tap = tap_name(&InterfaceIdentity { nic_index, @@ -955,16 +966,27 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res // A pass gets one answer about libvirt, not one per interface. Asking // again after it has failed is how a hung `libvirtd` turns a bounded // collection into an unbounded one. - let wanted = present || bindings.as_ref().is_some_and(|held| held.contains(&tap)); + let known_binding = bindings.as_ref().is_some_and(|held| held.contains(&tap)); + let wanted = present || known_binding; if libvirt && wanted && !is_macvtap(&tap) { if let Err(error) = delete_binding(libvirt_uri, &tap) { warn!(%tap, %error, "failed to remove an nwfilter binding"); libvirt = false; - first_error.get_or_insert(error); + // Unfiltered nodes do not require a running libvirtd. An + // unknown, possible binding must not make a successful TAP + // deletion fail there. Still retain failures for bindings we + // actually found, including ones left by an older policy. + if requires_binding || known_binding { + first_error.get_or_insert(error); + } } else if !present { info!(%tap, %vm_id, "removed orphaned nwfilter binding"); } } + if !libvirt && known_binding { + first_error + .get_or_insert_with(|| anyhow::anyhow!("nwfilter binding {tap} was not released")); + } if !present { continue; } @@ -2159,6 +2181,57 @@ mod tests { // and most of them miss. remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); } + /// Run with the same isolated-network-namespace setup as the test above. + #[test] + #[ignore = "needs root and its own network namespace"] + fn sweeps_without_libvirt_follow_the_nodes_filter_policy() { + assert!(nix::unistd::Uid::effective().is_root()); + assert_ne!( + std::fs::read_link("/proc/self/ns/net").unwrap(), + std::fs::read_link("/proc/1/ns/net").unwrap(), + "run this inside its own network namespace", + ); + let uri = "qemu:///nonexistent-for-this-test"; + for requires_binding in [false, true] { + let nic = identity("sweep-test", "vm-1", 0); + let tap = tap_name(&nic); + ip(&["tuntap", "add", "dev", &tap, "mode", "tap"]).unwrap(); + let mut config = NetdConfig { + libvirt_uri: uri.into(), + ..Default::default() + }; + config.network_filter = Some(NetworkFilterConfig { + mode: if requires_binding { + crate::config::NetworkFilterMode::Libvirt + } else { + crate::config::NetworkFilterMode::None + }, + ..Default::default() + }); + let sweep = || { + handle_request( + &config, + Request::RemoveAll { + instance_id: nic.instance_id.clone(), + vm_id: nic.vm_id.clone(), + }, + ) + }; + let result = sweep(); + // Even a binding failure must not leave the TAP on the bridge. + assert!(!Path::new("/sys/class/net").join(&tap).exists()); + if requires_binding { + assert!(result.is_err()); + // A retry cannot claim success just because the TAP is gone: + // its binding may still exist in the unreachable libvirt. + assert!(sweep().is_err()); + } else { + assert!(matches!(result.unwrap(), Outcome::Swept { removed: 1 })); + assert!(matches!(sweep().unwrap(), Outcome::Swept { removed: 0 })); + } + } + } + /// A netd that answers a sweep with no count did not sweep. Reading the /// absent field as zero is the same conflation `queues` is shaped to /// avoid, and here it would report a netd that cannot collect a VM's