From 104b5260517923837814180444597a4ba32961a1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 28 Aug 2026 07:24:20 -0700 Subject: [PATCH 01/34] feat(netd): carry host port mappings and the VM workdir to netd `port_map` is implemented as QEMU `hostfwd=` entries on a user-mode netdev. A bridge NIC has none, so `configure_networking` finds no interface to hang them on and emits none -- with no warning, no validation, and `GetInfo` still reporting the ports as though they worked. A VM moved from user mode to a bridge silently loses every published port. The VMM cannot fix that itself: it runs without CAP_NET_ADMIN by design, and the userspace forwarder that used to cover this was removed in e2e607fd4 because proxying on the host hands back the per-packet cost that leaving user mode was meant to escape. netd is privileged and is the only component that sees every VMM instance on a host, so it is also the only one that can arbitrate a host port between them. So state the requirement rather than assume it is met. `ingress` on `prepare_bridge` carries protocol, host address, host port and guest port. The host address is not decoration -- an admin port bound to loopback and a published one differ only there. Every field is caller-named, which is what `bridge`, `mac` and `queues` already get and the opposite of `filtered`. The line is whether netd can check what it is handed: an nwfilter name cannot be checked for whether it filters anything, while a host port is a closed space a node policy can be stated over. Naming is not deciding; which ports may be handed out stays netd's own configuration, exactly as `allowed_bridges` governs the bridge a caller names. The netd here builds interfaces and does not forward ports. It says so by leaving `ingress` out of its response, which is the reading `queues` already has: absent separates "this netd does not do that" from "nothing was asked for". The VMM warns on that rather than failing, because a VM deployed before this has been running with its ports dropped and refusing to launch it now would turn a silent misconfiguration into an outage on upgrade. Also add `workdir` to both prepare operations. Untrusted and never read for a decision, it is there so an operator reading netd's log can get from an opaque TAP name back to the VM that asked for it. No new operations, and nothing removed. --- docs/libvirt-network-filter.md | 12 +++ dstack/vmm/src/app.rs | 43 ++++++++ dstack/vmm/src/netd.rs | 181 +++++++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+) diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index 6cc2724cf..3bbd7645e 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -105,6 +105,18 @@ arguments. It never accepts a command, executable path, TAP name, or raw XML from a client. Filter XML is generated internally with XML escaping and is validated by libvirt. +A bridge prepare also carries two things `netd` does not need to build the TAP. +`workdir` names the VM's directory on the host: untrusted, never read for a +decision, and present only so an operator reading `netd`'s log can get from an +opaque TAP name back to the VM. `ingress` states the host ports the VM wants +reachable at its guest, which the VMM cannot arrange itself — it runs without +`CAP_NET_ADMIN` by design, and QEMU's `hostfwd=` entries need a user-mode netdev +that a bridge NIC does not have. The `netd` in this repository builds interfaces +and does not forward ports; it says so by leaving `ingress` out of its response, +the same reading `queues` gets, so a caller can tell "this netd does not do that" +from "nothing was asked for" instead of assuming ports were forwarded because a +TAP came back. + ## Deployment modes Production should run one shared service. `netd` reads the `[netd]` section, diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index e9205b07d..cf8eab211 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -564,6 +564,28 @@ impl App { return Ok(()); } let qemu_uid = Uid::effective().as_raw(); + // Only ever read back out of a log line: netd is told where the VM + // lives so an operator holding an opaque TAP name can reach the VM + // without going through the VMM first. + let workdir = self + .work_dir(&vm.manifest.id) + .map(|dir| dir.path().display().to_string()) + .unwrap_or_default(); + // `port_map` is implemented as QEMU `hostfwd=` entries on a user-mode + // netdev, so a bridge NIC drops every one of them. The VMM cannot + // forward them itself -- it runs without CAP_NET_ADMIN by design -- so + // it states the requirement and lets the node's netd answer it. + let ingress: Vec = vm + .manifest + .port_map + .iter() + .map(|mapping| netd::IngressRequest { + protocol: mapping.protocol.as_str().to_string(), + host_address: mapping.address.to_string(), + host_port: mapping.from, + guest_port: mapping.to, + }) + .collect(); let mut prepared = Vec::new(); for (nic_index, network) in networks.iter_mut().enumerate() { if !needs_netd_interface(network, &self.config.cvm) { @@ -593,6 +615,15 @@ impl App { // libvirt at all. filtered, queues, + workdir: workdir.clone(), + // Only the first NIC carries them, matching where user-mode + // networking puts its `hostfwd=` entries. Repeating the list + // would ask two interfaces to answer on one host port. + ingress: if nic_index == 0 { + ingress.clone() + } else { + Vec::new() + }, }), NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap(PrepareMacvtapRequest { identity: identity.clone(), @@ -601,6 +632,7 @@ impl App { qemu_uid, mode: network.macvtap_mode.clone(), queues, + workdir: workdir.clone(), }), NetworkingMode::User | NetworkingMode::Custom => continue, }; @@ -678,6 +710,17 @@ impl App { } Ok(()) })(); + // Ports asked for and not answered for used to vanish in silence: + // no warning, and `GetInfo` still listing them. A netd that forwards + // says what it built, so nothing said means nothing forwarded. + if nic_index == 0 && !ingress.is_empty() && response.ingress.is_none() { + warn!( + vm_id = %vm.manifest.id, + ports = ingress.len(), + "netd on this node does not forward host ports, so this VM's \ + port mappings do not apply to its bridge interface" + ); + } if let Err(error) = accepted { self.roll_back_prepared_networks(prepared).await; return Err(error); diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 6a760d1eb..2406d5502 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -69,6 +69,56 @@ pub struct PrepareBridgeRequest { /// rejects a device whose `IFF_MULTI_QUEUE` state differs from its own /// `queues=` argument, so this must match the launch exactly. pub queues: u32, + /// The VM's working directory on the host, for logs and diagnostics. + /// + /// Untrusted and never read for a decision: any process that can reach the + /// socket can assert anything here. It is carried so an operator reading + /// netd's log can get from an opaque TAP name back to the VM that asked for + /// it without going through the VMM. + #[serde(default)] + pub workdir: String, + /// Host ports this VM wants reachable at its guest. + /// + /// Empty asks for nothing, which is also what a caller predating the field + /// sends. Whether a netd forwards them is its own business; this states the + /// requirement rather than assuming it is met, and the response says what + /// was actually done. + #[serde(default)] + pub ingress: Vec, +} + +/// One host port a VM wants reachable at its guest. +/// +/// Every field is named by the caller, which is what `bridge`, `mac` and +/// `queues` already get and the opposite of `filtered`. The difference is +/// whether netd can check what it is handed: an nwfilter name cannot be checked +/// for whether it filters anything, while a host port is a closed space a node +/// policy can be stated over. Naming is not deciding -- which ports may be +/// handed out stays netd's own configuration, exactly as `allowed_bridges` +/// governs the bridge a caller names. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IngressRequest { + /// `"tcp"` or `"udp"`. + pub protocol: String, + /// Host address to accept on. Empty leaves the choice to netd. + /// + /// Not decoration: an admin port bound to loopback and a published one + /// differ only here. + #[serde(default)] + pub host_address: String, + /// Host port. Zero asks netd to choose one. + pub host_port: u16, + pub guest_port: u16, +} + +/// One forwarding rule a netd established, echoed so the caller can report what +/// the VM actually got rather than what it asked for. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IngressBinding { + pub protocol: String, + pub host_address: String, + pub host_port: u16, + pub guest_port: u16, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -84,6 +134,10 @@ pub struct PrepareMacvtapRequest { /// queues; QEMU then opens the character device once per queue. #[serde(default)] pub queues: u32, + /// The VM's working directory on the host. Informational only; see + /// [`PrepareBridgeRequest::workdir`]. + #[serde(default)] + pub workdir: String, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -120,6 +174,12 @@ struct Response { /// between "one queue was requested" and "this netd ignored the request". #[serde(default, skip_serializing_if = "Option::is_none")] queues: Option, + /// Forwarding rules netd established. Absent from a netd that does not + /// forward host ports, which is how the caller tells "nothing was asked + /// for" apart from "this request was ignored" -- the same reading `queues` + /// gets above. + #[serde(default, skip_serializing_if = "Option::is_none")] + ingress: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, } @@ -130,6 +190,7 @@ struct Prepared { tap: String, device: Option, queues: Option, + ingress: Option>, } impl Prepared { @@ -138,6 +199,7 @@ impl Prepared { tap, device: None, queues: None, + ingress: None, } } } @@ -162,6 +224,8 @@ pub fn instance_id(configured: &str, run_path: &Path) -> String { pub struct PreparedInterface { pub device: Option, pub queues: Option, + /// The forwarding rules netd established, if it forwards host ports at all. + pub ingress: Option>, } /// Marker carried in the error chain when the VMM could not reach netd at all. @@ -224,6 +288,7 @@ pub async fn request(socket: &Path, request: &Request) -> Result Resul tap: Some(prepared.tap), device: prepared.device, queues: prepared.queues, + ingress: prepared.ingress, error: None, }, Err(error) => { @@ -321,6 +387,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul tap: None, device: None, queues: None, + ingress: None, error: Some(format!("{error:#}")), } } @@ -459,6 +526,7 @@ fn prepare_macvtap( tap, device: Some(device), queues: Some(queues), + ingress: None, }) } Err(error) => { @@ -540,6 +608,10 @@ fn prepare_bridge( tap, device: None, queues: Some(queues), + // This netd builds interfaces; it is not the host's forwarder. Saying + // nothing here is what tells the caller that, so ports it asked for are + // reported as unmet rather than assumed done. + ingress: None, }) } @@ -868,6 +940,8 @@ mod tests { qemu_uid: 1000, filtered: true, queues: 0, + workdir: String::new(), + ingress: Vec::new(), }; let filter = NetworkFilterConfig { mode: crate::config::NetworkFilterMode::Libvirt, @@ -914,6 +988,8 @@ mod tests { qemu_uid: 1000, filtered: true, queues: 0, + workdir: String::new(), + ingress: Vec::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -983,6 +1059,8 @@ mod tests { qemu_uid: 1000, filtered: false, queues: 4, + workdir: String::new(), + ingress: Vec::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["queues"], 4); @@ -1015,6 +1093,7 @@ mod tests { qemu_uid: 1000, mode: "private".into(), queues: 0, + workdir: String::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_macvtap"); @@ -1032,6 +1111,8 @@ mod tests { qemu_uid: 1000, filtered: true, queues: 0, + workdir: String::new(), + ingress: Vec::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -1108,6 +1189,8 @@ mod tests { qemu_uid: 1000, filtered: false, queues: 4, + workdir: String::new(), + ingress: Vec::new(), }; let filtering = NetworkFilterConfig { mode: crate::config::NetworkFilterMode::Libvirt, @@ -1158,6 +1241,7 @@ mod tests { qemu_uid: 1000, mode: "bridge".into(), queues: 4, + workdir: String::new(), }; let error = match prepare_macvtap("test:///default", &request, &filtering) { Err(error) => error, @@ -1179,6 +1263,8 @@ mod tests { qemu_uid: 1000, filtered: true, queues: 1, + workdir: String::new(), + ingress: Vec::new(), }; // Nothing on the wire can name a filter: the field does not exist. let wire = serde_json::to_value(Request::PrepareBridge(request.clone())).unwrap(); @@ -1208,4 +1294,99 @@ mod tests { assert!(error.to_string().contains("timed out")); assert!(started.elapsed() < Duration::from_secs(2)); } + + #[test] + fn the_workdir_travels_but_older_callers_may_omit_it() { + let request = Request::PrepareBridge(PrepareBridgeRequest { + identity: identity("instance", "vm", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: true, + queues: 1, + workdir: "/opt/dstack/run/vm/vm".into(), + ingress: Vec::new(), + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["workdir"], "/opt/dstack/run/vm/vm"); + // It is a log line, not an input, so a caller that never sets it is not + // asking for anything different. + let Request::PrepareBridge(decoded) = decode_minimal_bridge() else { + panic!("expected a bridge prepare"); + }; + assert_eq!(decoded.workdir, ""); + } + + #[test] + fn host_ports_travel_with_the_bridge_prepare_and_default_to_none() { + let request = Request::PrepareBridge(PrepareBridgeRequest { + identity: identity("instance", "vm", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: true, + queues: 1, + workdir: String::new(), + ingress: vec![IngressRequest { + protocol: "udp".into(), + host_address: "0.0.0.0".into(), + host_port: 7483, + guest_port: 51820, + }], + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["ingress"][0]["protocol"], "udp"); + assert_eq!(value["ingress"][0]["host_port"], 7483); + assert_eq!(value["ingress"][0]["guest_port"], 51820); + // The bind address separates an admin port from a published one, so a + // forwarder that lost it would publish the admin port. + assert_eq!(value["ingress"][0]["host_address"], "0.0.0.0"); + + let Request::PrepareBridge(decoded) = decode_minimal_bridge() else { + panic!("expected a bridge prepare"); + }; + assert!(decoded.ingress.is_empty()); + } + + #[test] + fn saying_nothing_about_ports_is_how_a_netd_reports_it_forwards_none() { + // The same reading `queues` gets: absent distinguishes "this netd does + // not do that" from "nothing was asked for", so ports are never assumed + // forwarded just because the TAP came back. + let response: Response = serde_json::from_value(serde_json::json!({ + "ok": true, + "tap": "dt000000000000", + })) + .unwrap(); + assert!(response.ingress.is_none()); + + let response: Response = serde_json::from_value(serde_json::json!({ + "ok": true, + "tap": "dt000000000000", + "ingress": [{ + "protocol": "udp", + "host_address": "0.0.0.0", + "host_port": 7483, + "guest_port": 51820, + }], + })) + .unwrap(); + assert_eq!(response.ingress.unwrap().len(), 1); + } + + /// A prepare carrying only the fields that predate this change. + fn decode_minimal_bridge() -> Request { + serde_json::from_value(serde_json::json!({ + "operation": "prepare_bridge", + "instance_id": "instance", + "vm_id": "vm", + "nic_index": 0, + "bridge": "br0", + "mac": "02:00:00:00:00:01", + "qemu_uid": 1000, + "filtered": true, + "queues": 1, + })) + .unwrap() + } } From d9084f608d318c7d3b3aa871ea0c4bc41d12ee95 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 28 Aug 2026 08:41:25 -0700 Subject: [PATCH 02/34] feat(vmm): give port mappings a NIC, and carry them to netd Three things that are one thing: a port mapping had no way to say which NIC it uses, the NIC it silently got was often the wrong one, and on a bridge it got nothing at all. `port_map` is implemented as QEMU `hostfwd=` entries, and those need a user-mode netdev: let hostfwd_index = self.prepared.networks.iter() .position(|n| n.mode == NetworkingMode::User); Multi-NIC (#756) made that a choice, and it has been made silently ever since. A bridge NIC for external traffic beside a user-mode NIC for management -- the topology multi-NIC was added for -- puts every published port on the *management* NIC: the traffic reaches the guest, but over slirp, bypassing whatever the bridge NIC's nwfilter was there to enforce and hiding the client's address behind the slirp gateway. A second user-mode NIC can never publish anything, because `position` returns the first. And with no user-mode NIC at all there is no `hostfwd_index`, so every mapping is dropped with no warning while `GetInfo` keeps reporting the ports as though they worked. So say it. `PortMapping.nic_index` names the NIC a mapping enters through, `@` on the CLI, unset resolving to the first user-mode NIC and failing that the first bridge NIC. Existing VMs keep their behaviour exactly wherever a user-mode NIC exists; where none does, ports now have somewhere to go instead of nowhere. One mapping resolves to exactly one NIC, and that NIC's backend decides the mechanism: `hostfwd=` for user mode, netd for a bridge. Both sides read the same resolution, so no host port can be claimed twice. The VMM cannot forward a bridge NIC's ports itself -- it runs without CAP_NET_ADMIN by design, and the userspace forwarder that used to cover this was removed in e2e607fd4 because proxying on the host hands back the per-packet cost that leaving user mode was meant to escape. netd is privileged and is the only component that sees every VMM instance on a host, so it is also the only one that can arbitrate a host port between them. `ingress` on `prepare_bridge` carries the requirement there. Every field of it is caller-named, which is what `bridge`, `mac` and `queues` already get and the opposite of `filtered`. The line is whether netd can check what it is handed: an nwfilter name cannot be checked for whether it filters anything, while a host port is a closed space a node policy can be stated over. Naming is not deciding. The netd here builds interfaces and does not forward ports. It says so by leaving `ingress` out of its response -- the reading `queues` already has, where absent separates "this netd does not do that" from "nothing was asked for". The VMM warns rather than failing, because a VM deployed before this has been running with its ports dropped and refusing to launch it now would turn a silent misconfiguration into an outage. Also add `remove_all`. Teardown by identity only reaches the NIC indices its caller still has a record of, and that record is written after the interface exists: a VMM killed in between leaves a TAP nothing on disk points at, a lost record reads as "nothing to remove", and a manifest that lost a NIC leaves an index the list no longer reaches. netd derives every name a VM could occupy instead, bounded by what an identity may say. The VMM sweeps before preparing a launch as well as on stop, so a launch is self-healing regardless of what the record says, and an unreachable netd no longer fails a stop. And `workdir` on both prepare operations: untrusted, never read for a decision, there so an operator reading netd's log can get from an opaque TAP name back to the VM that asked for it. --- docs/bridge-networking.md | 24 +++++ docs/libvirt-network-filter.md | 10 ++- dstack/vmm/rpc/proto/vmm_rpc.proto | 9 ++ dstack/vmm/src/app.rs | 121 +++++++++++++++---------- dstack/vmm/src/app/network.rs | 128 +++++++++++++++++++++++++- dstack/vmm/src/app/qemu.rs | 33 +++---- dstack/vmm/src/app/vm_info.rs | 1 + dstack/vmm/src/main_service.rs | 32 ++++++- dstack/vmm/src/netd.rs | 140 +++++++++++++++++++++++++++-- dstack/vmm/src/vmm-cli.py | 29 ++++-- 10 files changed, 443 insertions(+), 84 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 6bf3e6d5e..5724e719d 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -199,6 +199,30 @@ The remaining bytes are derived from the VM ID hash. The prefix applies to all n - Docker's nftables chains (`DOCKER-FORWARD`) run before libvirt's but do not block virbr0 traffic - Use `setup-bridge.sh check --bridge ` to diagnose missing rules +### Which NIC a port mapping uses + +A port mapping says which NIC its traffic enters through: + +```bash +vmm-cli.py deploy ... --port udp:0.0.0.0:7483:51820@0 --port tcp:127.0.0.1:7484:8001@0 +``` + +Leave `@` off and the VMM picks the first user-mode NIC — where QEMU's +`hostfwd=` entries have always gone — and failing that the first bridge NIC. A +single-NIC VM never needs it. + +With several NICs the choice used to be made silently, and not always the way an +operator would have. A bridge NIC for external traffic beside a user-mode NIC for +management — the topology multi-NIC was added for — put every published port on +the *management* NIC: the traffic reached the guest, but over slirp, bypassing +whatever the bridge NIC's nwfilter was there to enforce and hiding the client's +address behind the slirp gateway. A second user-mode NIC could never publish +anything at all, because only the first was ever selected. + +A mapping resolves to exactly one NIC, and that NIC's backend decides the +mechanism: `hostfwd=` for user mode, `netd` for a bridge. Nothing can be claimed +by both. + ### Mixing networking modes Bridge and user-mode VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index 3bbd7645e..e2ee77778 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -105,10 +105,18 @@ arguments. It never accepts a command, executable path, TAP name, or raw XML from a client. Filter XML is generated internally with XML escaping and is validated by libvirt. +Teardown by identity only reaches the NIC indices its caller still has a record +of, and that record is written *after* the interface exists — a VMM killed in +between leaves a TAP nothing on disk points at, and a manifest that lost a NIC +leaves the same thing behind. `remove_all` names a VM instead of an interface +and derives every name that VM could occupy, so neither has to be recorded for +teardown to work. The VMM sweeps before preparing a launch as well as on stop, +which makes a launch self-healing regardless of what the record says. + A bridge prepare also carries two things `netd` does not need to build the TAP. `workdir` names the VM's directory on the host: untrusted, never read for a decision, and present only so an operator reading `netd`'s log can get from an -opaque TAP name back to the VM. `ingress` states the host ports the VM wants +opaque TAP name back to the VM. `ingress` states the host ports that NIC should make reachable at its guest, which the VMM cannot arrange itself — it runs without `CAP_NET_ADMIN` by design, and QEMU's `hostfwd=` entries need a user-mode netdev that a bridge NIC does not have. The `netd` in this repository builds interfaces diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 31fef51f4..239ac2d79 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -182,6 +182,15 @@ message PortMapping { uint32 vm_port = 3; // Host address string host_address = 4; + // Which NIC this mapping's traffic enters through, as an index into + // `networks`. Unset picks the first user-mode NIC, which is where QEMU's + // hostfwd entries have always gone, and failing that the first bridge NIC. + // + // A VM with one NIC never needs it. With several there is a choice, and it + // used to be made silently: a bridge NIC for external traffic beside a + // user-mode NIC for management -- the topology multi-NIC was added for -- + // put every published port on the management NIC. + optional uint32 nic_index = 5; } // Partial configuration used when mutating an existing VM. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index cf8eab211..528cd44b9 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -46,9 +46,9 @@ use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; pub(crate) use network::{ - clamp_queues_without_netd, filters_bridge_traffic, needs_netd_interface, netd_available, - netd_teardown, resolve_networking, resolved_networks, settle_vhost, validate_resolved_network, - validate_resolved_networks, + clamp_queues_without_netd, filters_bridge_traffic, ingress_for, needs_netd_interface, + netd_available, netd_teardown, resolve_networking, resolved_networks, settle_vhost, + validate_resolved_network, validate_resolved_networks, }; pub use qemu::VmConfig; // Exported so the RPC layer can assert that everything it reports is @@ -97,6 +97,10 @@ pub struct PortMapping { pub protocol: Protocol, pub from: u16, pub to: u16, + /// Which NIC carries this mapping. `None` resolves by the node's rule; see + /// [`crate::app::network::ingress_nic`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nic_index: Option, } /// An extra disk attached to the VM (e.g. a pre-baked verity volume). `source` @@ -563,6 +567,27 @@ impl App { { return Ok(()); } + // Whatever an earlier boot left behind, from a crash between creating + // an interface and recording it or from a NIC this VM no longer has. + // Prepare replaces the names it is about to use, but only those; an + // index nothing will claim again is only reachable from here. + if let Err(error) = netd::remove_all( + &self.config.netd.socket, + &self.config.cvm.instance_id, + &vm.manifest.id, + ) + .await + { + if !netd::is_unreachable(&error) { + warn!(vm_id = %vm.manifest.id, %error, "failed to sweep stale netd interfaces"); + } + } + // Resolved before the loop borrows `networks` mutably, and once rather + // than per NIC, so both the request and the warning below read the same + // answer. + let ingress: Vec> = (0..networks.len()) + .map(|nic_index| ingress_for(&vm.manifest.port_map, networks, nic_index)) + .collect(); let qemu_uid = Uid::effective().as_raw(); // Only ever read back out of a log line: netd is told where the VM // lives so an operator holding an opaque TAP name can reach the VM @@ -575,17 +600,6 @@ impl App { // netdev, so a bridge NIC drops every one of them. The VMM cannot // forward them itself -- it runs without CAP_NET_ADMIN by design -- so // it states the requirement and lets the node's netd answer it. - let ingress: Vec = vm - .manifest - .port_map - .iter() - .map(|mapping| netd::IngressRequest { - protocol: mapping.protocol.as_str().to_string(), - host_address: mapping.address.to_string(), - host_port: mapping.from, - guest_port: mapping.to, - }) - .collect(); let mut prepared = Vec::new(); for (nic_index, network) in networks.iter_mut().enumerate() { if !needs_netd_interface(network, &self.config.cvm) { @@ -616,14 +630,11 @@ impl App { filtered, queues, workdir: workdir.clone(), - // Only the first NIC carries them, matching where user-mode - // networking puts its `hostfwd=` entries. Repeating the list - // would ask two interfaces to answer on one host port. - ingress: if nic_index == 0 { - ingress.clone() - } else { - Vec::new() - }, + // Only the mappings that resolve to this NIC. One mapping + // lands on exactly one, and a user-mode NIC's are emitted + // as QEMU `hostfwd=` instead, so no host port is claimed + // twice. + ingress: ingress[nic_index].clone(), }), NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap(PrepareMacvtapRequest { identity: identity.clone(), @@ -713,10 +724,11 @@ impl App { // Ports asked for and not answered for used to vanish in silence: // no warning, and `GetInfo` still listing them. A netd that forwards // says what it built, so nothing said means nothing forwarded. - if nic_index == 0 && !ingress.is_empty() && response.ingress.is_none() { + let asked = ingress[nic_index].len(); + if asked > 0 && response.ingress.is_none() { warn!( vm_id = %vm.manifest.id, - ports = ingress.len(), + ports = asked, "netd on this node does not forward host ports, so this VM's \ port mappings do not apply to its bridge interface" ); @@ -843,40 +855,51 @@ impl App { } } + /// Deletes every host interface netd holds for this VM. + /// + /// A sweep rather than one removal per recorded NIC. The record is written + /// after the interface exists, so a VMM killed in between leaves a TAP + /// nothing on disk points at; a lost or unreadable record reads as an empty + /// list, which used to mean "nothing to remove"; and a manifest that lost a + /// NIC leaves an index the list no longer reaches. netd derives the names + /// instead, so none of that has to be true for teardown to work. + /// + /// `networks` now only decides whether to ask at all. An unreachable netd + /// is not a failure: most nodes run none, and stopping a VM must not depend + /// on one being up. pub(crate) async fn remove_filtered_networks( &self, vm_id: &str, networks: &[Networking], ) -> Result<()> { - if networks - .iter() - .all(|network| netd_teardown(network, &self.config.cvm).is_none()) - { + // An empty list is not "no interfaces", it is "no record" -- exactly + // the case a sweep exists for. A record that names only backends netd + // never touches is the one case worth skipping. + let recorded_none = !networks.is_empty() + && networks + .iter() + .all(|network| netd_teardown(network, &self.config.cvm).is_none()); + if recorded_none { return Ok(()); } - let mut first_error = None; - for (nic_index, network) in networks.iter().enumerate().rev() { - let Some(filtered) = netd_teardown(network, &self.config.cvm) else { - continue; - }; - let identity = InterfaceIdentity { - instance_id: self.config.cvm.instance_id.clone(), - vm_id: vm_id.to_string(), - nic_index, - }; - if let Err(error) = netd::request( - &self.config.netd.socket, - &NetdRequest::Remove { identity, filtered }, - ) - .await - { - first_error.get_or_insert(error); + match netd::remove_all( + &self.config.netd.socket, + &self.config.cvm.instance_id, + vm_id, + ) + .await + { + Ok(0) => Ok(()), + Ok(removed) => { + info!(vm_id, removed, "removed netd-managed interfaces"); + Ok(()) } + Err(error) if netd::is_unreachable(&error) => { + debug!(vm_id, %error, "no netd to remove interfaces from"); + Ok(()) + } + Err(error) => Err(error).context("failed to remove netd-managed networking"), } - if let Some(error) = first_error { - return Err(error).context("failed to remove netd-managed networking"); - } - Ok(()) } pub(crate) async fn stop_vm_process(&self, id: &str) -> Result<()> { diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index 4017e89ff..0ee7c4e38 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -9,7 +9,7 @@ use std::path::Path; use anyhow::{bail, Result}; use sha2::{Digest, Sha256}; -use super::Manifest; +use super::{Manifest, PortMapping}; use crate::config::{ CvmConfig, NetdInterface, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, MAX_NET_QUEUES, @@ -329,6 +329,54 @@ pub(crate) fn warn_if_vhost_net_missing(networks: &[Networking]) { } } +/// Which NIC an unpinned port mapping's traffic enters through. +/// +/// The first user-mode NIC, because that is where QEMU's `hostfwd=` entries +/// have always gone and existing VMs must keep behaving the same way; failing +/// that the first bridge NIC, which is the only other backend with a path into +/// the guest. `macvtap` bypasses the host bridge and `custom` owns its own +/// netdev string, so neither can carry one. +pub(crate) fn default_ingress_nic(networks: &[Networking]) -> Option { + networks + .iter() + .position(|network| network.nic.mode == NetworkingMode::User) + .or_else(|| { + networks + .iter() + .position(|network| network.nic.mode == NetworkingMode::Bridge) + }) +} + +/// Which NIC a port mapping's traffic enters through. +/// +/// One mapping resolves to at most one NIC, and that NIC's backend decides the +/// mechanism: `hostfwd=` for user mode, netd for a bridge. That is what keeps +/// QEMU and netd from both claiming one host port. +pub(crate) fn ingress_nic(mapping: &PortMapping, networks: &[Networking]) -> Option { + mapping + .nic_index + .or_else(|| default_ingress_nic(networks)) + .filter(|index| *index < networks.len()) +} + +/// The host ports one NIC carries, as netd requests. +pub(crate) fn ingress_for( + port_map: &[PortMapping], + networks: &[Networking], + nic_index: usize, +) -> Vec { + port_map + .iter() + .filter(|mapping| ingress_nic(mapping, networks) == Some(nic_index)) + .map(|mapping| crate::netd::IngressRequest { + protocol: mapping.protocol.as_str().to_string(), + host_address: mapping.address.to_string(), + host_port: mapping.from, + guest_port: mapping.to, + }) + .collect() +} + /// Derives a deterministic, locally administered unicast MAC address. /// /// Index zero preserves the legacy single-NIC derivation. Later interfaces @@ -356,10 +404,12 @@ pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) #[cfg(test)] mod tests { use super::{ - clamp_queues_without_netd, effective_vhost, mac_address_for_vm_index, needs_netd_interface, - netd_teardown, resolve_networking, resolved_networks, settle_vhost, - validate_resolved_networks, + clamp_queues_without_netd, default_ingress_nic, effective_vhost, ingress_for, ingress_nic, + mac_address_for_vm_index, needs_netd_interface, netd_teardown, resolve_networking, + resolved_networks, settle_vhost, validate_resolved_networks, }; + use crate::app::PortMapping; + use crate::config::Protocol; use crate::config::{Networking, NetworkingMode, NicNetworking}; fn macvtap_network() -> NicNetworking { @@ -722,4 +772,74 @@ mod tests { "c6:74:2c:65:14:b9" ); } + + fn nic(mode: NetworkingMode) -> Networking { + Networking { + nic: NicNetworking { + mode, + ..NicNetworking::default() + }, + ..Networking::default() + } + } + + fn mapping(host_port: u16, nic_index: Option) -> PortMapping { + PortMapping { + address: "0.0.0.0".parse().unwrap(), + protocol: Protocol::Tcp, + from: host_port, + to: host_port, + nic_index, + } + } + + #[test] + fn an_unpinned_mapping_still_lands_where_hostfwd_always_put_it() { + // Existing VMs must not move. QEMU's `hostfwd=` has always gone to the + // first user-mode NIC, so that stays the answer wherever there is one. + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; + assert_eq!(default_ingress_nic(&networks), Some(1)); + assert_eq!(ingress_nic(&mapping(443, None), &networks), Some(1)); + + // With no user-mode NIC there was nowhere at all, which is the hole + // this closes: a bridge NIC is the only other backend with a path. + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::Bridge)]; + assert_eq!(default_ingress_nic(&networks), Some(0)); + + // macvtap bypasses the host bridge and custom owns its netdev string. + let networks = [nic(NetworkingMode::Macvtap), nic(NetworkingMode::Custom)]; + assert_eq!(default_ingress_nic(&networks), None); + assert_eq!(ingress_nic(&mapping(443, None), &networks), None); + } + + #[test] + fn a_pinned_mapping_goes_where_it_says() { + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; + assert_eq!(ingress_nic(&mapping(443, Some(0)), &networks), Some(0)); + // Out of range resolves to nothing rather than to something arbitrary. + // Deployment refuses it outright; a manifest that lost a NIC lands here. + assert_eq!(ingress_nic(&mapping(443, Some(7)), &networks), None); + } + + #[test] + fn one_mapping_reaches_exactly_one_nic() { + // The property that keeps QEMU and netd from both claiming a host port: + // every mapping appears under one NIC and no other. + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; + let port_map = [ + mapping(443, Some(0)), + mapping(8080, None), + mapping(9090, Some(1)), + ]; + let per_nic: Vec<_> = (0..networks.len()) + .map(|index| ingress_for(&port_map, &networks, index)) + .collect(); + // Only NIC 0 is a bridge, so only its list becomes netd requests; the + // other two ride QEMU's hostfwd on NIC 1. + assert_eq!(per_nic[0].len(), 1); + assert_eq!(per_nic[0][0].host_port, 443); + assert_eq!(per_nic[1].len(), 2); + let total: usize = per_nic.iter().map(Vec::len).sum(); + assert_eq!(total, port_map.len()); + } } diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 94a6348fe..828230068 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -10,8 +10,8 @@ use super::{ image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, network::{ - bridge_helper, mac_address_for_vm_index, needs_netd_interface, validate_resolved_networks, - warn_if_vhost_net_missing, + bridge_helper, ingress_nic, mac_address_for_vm_index, needs_netd_interface, + validate_resolved_networks, warn_if_vhost_net_missing, }, pci_numa_node, round_up, GpuConfig, VmWorkDir, }; @@ -631,11 +631,6 @@ impl QemuCommandBuilder<'_> { fn configure_networking(&self, command: &mut Command) -> Result<()> { let macvtap_fds = macvtap_fd_layout(&self.prepared.networks); - let hostfwd_index = self - .prepared - .networks - .iter() - .position(|networking| networking.nic.mode == NetworkingMode::User); for (index, networking) in self.prepared.networks.iter().enumerate() { let net_id = format!("net{index}"); let mac = mac_address_for_vm_index( @@ -663,16 +658,21 @@ impl QemuCommandBuilder<'_> { networking.dhcp_start, if networking.restrict { "yes" } else { "no" } ); - if hostfwd_index == Some(index) { - for mapping in &self.vm.manifest.port_map { - netdev.push_str(&format!( - ",hostfwd={}:{}:{}-:{}", - mapping.protocol.as_str(), - mapping.address, - mapping.from, - mapping.to - )); + // Only the mappings that resolve to this NIC. A mapping + // lands on exactly one, and that NIC's backend decides the + // mechanism, so a bridge NIC's ports go to netd instead of + // being claimed here as well. + for mapping in &self.vm.manifest.port_map { + if ingress_nic(mapping, &self.prepared.networks) != Some(index) { + continue; } + netdev.push_str(&format!( + ",hostfwd={}:{}:{}-:{}", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.to + )); } netdev } @@ -1192,6 +1192,7 @@ mod tests { protocol: Protocol::Tcp, from: 18080, to: 8080, + nic_index: None, }], created_at_ms: 0, hugepages: false, diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 19d72118a..4cd10762b 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -207,6 +207,7 @@ impl VmInfo { .port_map .iter() .map(|mapping| pb::PortMapping { + nic_index: mapping.nic_index.map(|index| index as u32), protocol: mapping.protocol.as_str().into(), host_address: mapping.address.to_string(), host_port: mapping.from as u32, diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 6f9a0905d..158b49c10 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -168,6 +168,28 @@ fn port_mappings_conflict(left: &PortMapping, right: &PortMapping) -> bool { || right.address.is_unspecified()) } +/// Rejects a mapping pinned to a NIC the VM does not have. +/// +/// Range only. Whether the named NIC's backend can carry a host port is +/// resolved at launch, where the node configuration that decides it is the one +/// in force -- and where an existing VM gets a warning rather than a refusal. +fn validate_port_mapping_nics(mappings: &[PortMapping], nic_count: usize) -> Result<()> { + for mapping in mappings { + let Some(index) = mapping.nic_index else { + continue; + }; + if index >= nic_count { + bail!( + "port mapping {} {}:{} names NIC {index}, but this VM has {nic_count}", + mapping.protocol.as_str(), + mapping.address, + mapping.from + ); + } + } + Ok(()) +} + fn validate_unique_port_mappings(mappings: &[PortMapping]) -> Result<()> { for (index, mapping) in mappings.iter().enumerate() { if mappings[..index] @@ -216,10 +238,14 @@ pub fn create_manifest_from_vm_config( protocol, from, to, + nic_index: p.nic_index.map(|index| index as usize), }) }) .collect::>>()?; validate_unique_port_mappings(&port_map)?; + let networks = networks_from_vm_config(&request, cvm_config)?; + // An empty list inherits the node default, which is one NIC. + validate_port_mapping_nics(&port_map, networks.len().max(1))?; let app_id = match &request.app_id { Some(id) => id.strip_prefix("0x").unwrap_or(id).to_lowercase(), @@ -268,7 +294,7 @@ pub fn create_manifest_from_vm_config( no_tee: request.no_tee || simulated_tee.is_some(), simulated_tee, swtpm, - networks: networks_from_vm_config(&request, cvm_config)?, + networks, volumes, }) } @@ -918,6 +944,7 @@ impl VmmRpc for RpcHandler { protocol: p.protocol.parse().context("Invalid protocol")?, from: p.host_port.try_into().context("Invalid host port")?, to: p.vm_port.try_into().context("Invalid vm port")?, + nic_index: p.nic_index.map(|index| index as usize), }) }) .collect::>>()?; @@ -960,6 +987,9 @@ impl VmmRpc for RpcHandler { } manifest.networks = networks; } + // After both, since either half can move and the other still has to + // agree with it. + validate_port_mapping_nics(&manifest.port_map, manifest.networks.len().max(1))?; let compose_file = fs::read_to_string(vm_work_dir.app_compose_path()) .context("failed to read app compose for swtpm decision")?; manifest.swtpm = needs_swtpm( diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 2406d5502..024dc7db0 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -43,6 +43,10 @@ const LOCK_PATH: &str = "/run/lock/dstack-netd.lock"; /// Upper bound on TAP queue pairs netd will create. Mirrors the VMM's own cap /// so a malformed request cannot ask the kernel for an unbounded device. const MAX_QUEUES: u32 = 64; +/// Highest NIC index an identity may name. Also the width of the space a +/// whole-VM sweep has to enumerate, since it derives names instead of reading a +/// record. +const MAX_NIC_INDEX: usize = 255; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InterfaceIdentity { @@ -153,6 +157,18 @@ pub enum Request { /// trusting this field. filtered: bool, }, + /// Delete every interface netd holds for one VM. + /// + /// Teardown by identity can only reach the NIC indices its caller still has + /// a record of, and that record is written after the interface exists: a + /// VMM killed in between leaves a TAP nothing on disk points at. A manifest + /// that lost a NIC leaves the same thing behind. Both are found here + /// without a record, because every name netd can produce for a VM is + /// derivable from its identity. + RemoveAll { + instance_id: String, + vm_id: String, + }, /// Verify a deterministic TAP and binding for operations and integration /// diagnostics. The VMM startup path uses Prepare rather than Check. Check { @@ -180,6 +196,9 @@ struct Response { /// gets above. #[serde(default, skip_serializing_if = "Option::is_none")] ingress: Option>, + /// How many interfaces a whole-VM sweep deleted. + #[serde(default, skip_serializing_if = "Option::is_none")] + removed: Option, #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, } @@ -191,6 +210,7 @@ struct Prepared { device: Option, queues: Option, ingress: Option>, + removed: Option, } impl Prepared { @@ -200,6 +220,18 @@ impl Prepared { device: None, queues: None, ingress: None, + removed: None, + } + } + + /// A sweep names no single interface, so it reports how many it deleted. + fn removed(removed: usize) -> Self { + Self { + tap: String::new(), + device: None, + queues: None, + ingress: None, + removed: Some(removed), } } } @@ -250,10 +282,36 @@ pub fn is_unreachable(error: &anyhow::Error) -> bool { } pub async fn request(socket: &Path, request: &Request) -> Result { + let response = exchange(socket, request).await?; + if response.tap.as_deref().unwrap_or_default().is_empty() { + bail!("netd response omitted TAP name"); + } + Ok(PreparedInterface { + device: response.device, + queues: response.queues, + ingress: response.ingress, + }) +} + +/// Deletes every interface netd holds for one VM, returning how many there +/// were. See [`Request::RemoveAll`]. +pub async fn remove_all(socket: &Path, instance_id: &str, vm_id: &str) -> Result { + let request = Request::RemoveAll { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + }; + Ok(exchange(socket, &request) + .await? + .removed + .unwrap_or_default()) +} + +async fn exchange(socket: &Path, request: &Request) -> Result { let operation = match request { Request::PrepareBridge(_) => "prepare_bridge", Request::PrepareMacvtap(_) => "prepare_macvtap", Request::Remove { .. } => "remove", + Request::RemoveAll { .. } => "remove_all", Request::Check { .. } => "check", }; let exchange = async { @@ -284,12 +342,7 @@ pub async fn request(socket: &Path, request: &Request) -> Result Resul device: prepared.device, queues: prepared.queues, ingress: prepared.ingress, + removed: prepared.removed, error: None, }, Err(error) => { @@ -388,6 +442,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul device: None, queues: None, ingress: None, + removed: None, error: Some(format!("{error:#}")), } } @@ -426,6 +481,10 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { Request::PrepareMacvtap(request) => { prepare_macvtap(libvirt_uri, &request, config.filter_policy()) } + Request::RemoveAll { instance_id, vm_id } => { + let removed = sweep_vm_interfaces(libvirt_uri, &instance_id, &vm_id)?; + Ok(Prepared::removed(removed)) + } Request::Remove { identity, filtered } => { validate_identity(&identity)?; let tap = tap_name(&identity); @@ -527,6 +586,7 @@ fn prepare_macvtap( device: Some(device), queues: Some(queues), ingress: None, + removed: None, }) } Err(error) => { @@ -612,6 +672,7 @@ fn prepare_bridge( // nothing here is what tells the caller that, so ports it asked for are // reported as unmet rather than assumed done. ingress: None, + removed: None, }) } @@ -629,6 +690,49 @@ enum BindingCleanup { BestEffort, } +/// Deletes every interface a VM could hold, by deriving each name rather than +/// consulting a record. +/// +/// `validate_identity` caps the NIC index, so the whole space a VM can occupy +/// is enumerable: 256 names, each a `stat` that usually misses. Cleanup is +/// best-effort about bindings -- nothing is about to take these names, and a +/// node running unfiltered TAPs need not have libvirtd at all. +fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Result { + let identity = InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + nic_index: 0, + }; + validate_identity(&identity)?; + let mut removed = 0; + let mut first_error = None; + for nic_index in 0..=MAX_NIC_INDEX { + let tap = tap_name(&InterfaceIdentity { + nic_index, + ..identity.clone() + }); + if !Path::new("/sys/class/net").join(&tap).exists() { + continue; + } + match remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort) { + // Keep going after a failure. Stopping at the first one would leave + // the rest of a VM's interfaces behind over one that is stuck. + Err(error) => { + warn!(%tap, %error, "failed to remove interface"); + first_error.get_or_insert(error); + } + Ok(()) => { + info!(%tap, %vm_id, "removed interface"); + removed += 1; + } + } + } + match first_error { + Some(error) => Err(error).context("failed to remove every interface for this VM"), + None => Ok(removed), + } +} + fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Result<()> { let macvtap = is_macvtap(tap); if Path::new("/sys/class/net").join(tap).exists() { @@ -766,7 +870,7 @@ fn validate_identity(identity: &InterfaceIdentity) -> Result<()> { bail!("invalid {label}"); } } - if identity.nic_index > 255 { + if identity.nic_index > MAX_NIC_INDEX { bail!("NIC index is out of range"); } Ok(()) @@ -1389,4 +1493,26 @@ mod tests { })) .unwrap() } + + #[test] + fn a_whole_vm_sweep_needs_no_record_of_what_it_is_deleting() { + let value = serde_json::to_value(Request::RemoveAll { + instance_id: "instance".into(), + vm_id: "vm".into(), + }) + .unwrap(); + assert_eq!(value["operation"], "remove_all"); + assert_eq!(value["instance_id"], "instance"); + assert_eq!(value["vm_id"], "vm"); + // No NIC index: the point is reaching the ones the caller can no longer + // name, so it names none and netd derives the whole space instead. + assert!(value.get("nic_index").is_none()); + + // That space is bounded by what an identity may say, which is what + // makes deriving it cheap enough to do on every launch. + let mut identity = identity("instance", "vm", MAX_NIC_INDEX); + assert!(validate_identity(&identity).is_ok()); + identity.nic_index = MAX_NIC_INDEX + 1; + assert!(validate_identity(&identity).is_err()); + } } diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 20e515520..faebc9d0f 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -321,17 +321,31 @@ def encrypt_env(envs, hex_public_key: str) -> str: def parse_port_mapping(port_str: str) -> Dict: - """Parse a port mapping string into a dictionary.""" + """Parse a port mapping string into a dictionary. + + Accepts an optional "@" suffix naming which NIC the traffic enters + through. Without it the VMM picks: the first user-mode NIC, else the first + bridge NIC. A single-NIC VM never needs it. + """ + nic_index = None + if "@" in port_str: + port_str, _, nic = port_str.rpartition("@") + try: + nic_index = int(nic) + except ValueError: + raise argparse.ArgumentTypeError(f"Invalid NIC index: {nic}") + if nic_index < 0: + raise argparse.ArgumentTypeError(f"Invalid NIC index: {nic}") parts = port_str.split(":") if len(parts) == 3: - return { + mapping = { "protocol": parts[0], "host_address": "127.0.0.1", "host_port": int(parts[1]), "vm_port": int(parts[2]), } elif len(parts) == 4: - return { + mapping = { "protocol": parts[0], "host_address": parts[1], "host_port": int(parts[2]), @@ -339,6 +353,9 @@ def parse_port_mapping(port_str: str) -> Dict: } else: raise argparse.ArgumentTypeError(f"Invalid port mapping format: {port_str}") + if nic_index is not None: + mapping["nic_index"] = nic_index + return mapping def read_utf8(filepath: str) -> str: @@ -1907,7 +1924,7 @@ def _patched_format_help(): "--port", action="append", type=str, - help="Port mapping in format: protocol[:address]:from:to", + help="Port mapping in format: protocol[:address]:from:to[@nic]", ) deploy_parser.add_argument( "--gpu", @@ -2063,7 +2080,7 @@ def _patched_format_help(): action="append", type=str, required=True, - help="Port mapping in format: protocol[:address]:from:to (can be used multiple times)", + help="Port mapping in format: protocol[:address]:from:to[@nic] (can be used multiple times)", ) # Update (all-in-one) command @@ -2133,7 +2150,7 @@ def _patched_format_help(): "--port", action="append", type=str, - help="Port mapping in format: protocol[:address]:from:to (can be used multiple times)", + help="Port mapping in format: protocol[:address]:from:to[@nic] (can be used multiple times)", ) port_group.add_argument( "--no-ports", From f13e0ef6c8ac53757344d9c4a1446ab580b3a1ff Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 30 Aug 2026 23:05:54 -0700 Subject: [PATCH 03/34] fixup! feat(vmm): give port mappings a NIC, and carry them to netd --- docs/bridge-networking.md | 42 +- docs/libvirt-network-filter.md | 26 +- docs/network-data-plane.md | 51 +-- docs/onboarding.md | 2 +- dstack/crates/dstack-cli-core/src/ports.rs | 38 ++ dstack/crates/dstack-cli/src/main.rs | 3 +- dstack/crates/dstackup/src/install.rs | 3 + dstack/vmm/src/app.rs | 245 ++++++++---- dstack/vmm/src/app/network.rs | 371 +++++++----------- dstack/vmm/src/app/qemu.rs | 124 +++--- dstack/vmm/src/app/vm_info.rs | 13 +- dstack/vmm/src/config.rs | 48 +-- dstack/vmm/src/main_service.rs | 126 +++++- dstack/vmm/src/netd.rs | 121 +++++- dstack/vmm/src/one_shot.rs | 50 +-- dstack/vmm/src/vmm-cli.py | 9 +- .../vmm/ui/src/components/CreateVmDialog.ts | 5 +- .../ui/src/components/PortMappingEditor.ts | 17 + dstack/vmm/ui/src/composables/useVmManager.ts | 17 + dstack/vmm/vmm.toml | 21 +- 20 files changed, 740 insertions(+), 592 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 5724e719d..4c763fc2f 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -143,35 +143,37 @@ mode = "bridge" bridge = "dstack-br0" ``` -### QEMU bridge helper setup (needed unless every bridge NIC goes through netd) +### netd is required -The bridge helper allows QEMU to create and attach TAP devices without VMM needing root privileges. -It is used only on the single-queue bridge paths; a NIC that `netd` builds never touches it, so a -node that runs `netd` for all of its bridge VMs does not need it at all. - -The VMM probes `/usr/lib/qemu/qemu-bridge-helper`, `/usr/libexec/qemu-bridge-helper` and -`/usr/local/libexec/qemu-bridge-helper`. Set `cvm.qemu_bridge_helper` in `vmm.toml` for a path -outside that list. +Bridge networking needs `netd`, the privileged helper that owns every host +interface a bridge or macvtap NIC uses. It is the same binary: ```bash -# Allow QEMU to use the bridge -sudo mkdir -p /etc/qemu -echo "allow virbr0" | sudo tee /etc/qemu/bridge.conf -# Or for manual bridge: echo "allow dstack-br0" | sudo tee /etc/qemu/bridge.conf - -# Set setuid on bridge helper -sudo chmod u+s /usr/lib/qemu/qemu-bridge-helper +sudo dstack-vmm netd -c vmm.toml ``` +Nothing else on the node needs `CAP_NET_ADMIN`: the VMM itself still runs +unprivileged, and `netd` holds the privilege behind a Unix socket whose +filesystem permissions authorize callers. + +This used to be conditional — `netd` built the TAP when libvirt filtering was on +or when the NIC wanted more than one queue pair, and otherwise QEMU's setuid +`qemu-bridge-helper` did. Two owners meant two answers to the same questions: +which netdev QEMU gets, whether vhost is really on, and where a bridge NIC's +published ports go. Only `netd` can answer the last one, because only `netd` +sees every VMM instance on the host and can arbitrate a port between them. So a +bridge NIC's host interface has one owner now, on every node. + +`qemu-bridge-helper` is no longer used, and `/etc/qemu/bridge.conf` no longer +needs an `allow` line for the bridge. + ## How it works -- With more than one queue pair, or with libvirt filtering on, `netd` creates the TAP and the VMM passes `-netdev tap,id=net0,ifname=,...` — this is the usual case on a node running `netd` with multi-vCPU VMs, since queue pairs default to the VM's vCPU count. Without `netd`, a bridge NIC that took that default drops back to one queue pair and takes a helper path below -- Otherwise the VMM passes `-netdev tap,id=net0,br=,helper=,vhost=on`, or `-netdev bridge,id=net0,br=` when vhost is off or no helper is found -- QEMU's bridge helper (setuid) creates a TAP device and attaches it to the bridge on the two helper paths +- `netd` creates a persistent TAP, attaches it to the bridge, binds the nwfilter if the node filters, and the VMM passes `-netdev tap,id=net0,ifname=,...` - Guest MAC address is derived from SHA256 of the VM ID, with an optional configurable prefix (stable across restarts for DHCP IP consistency) - The host DHCP server (dnsmasq) assigns an IP to the VM -- On the two bridge-helper paths the TAP disappears when QEMU exits; a `netd`-created TAP is persistent and is deleted when the VMM tears the VM's networking down -- The VMM process itself needs neither root nor `CAP_NET_ADMIN` on any path; the `netd` path moves that privilege into a separate root service instead +- The TAP outlives QEMU and is deleted when the VMM tears the VM's networking down, so a VM that crashes does not leave its filter rules attached to a name the next VM could take +- The VMM process needs neither root nor `CAP_NET_ADMIN`; `netd` holds that privilege in a separate service ### MAC address prefix diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index e2ee77778..4ba827ec5 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -14,11 +14,10 @@ host mechanism. The measurable acceptance criteria are: -- `network_filter = "none"` installs no nwfilter binding. It still uses `netd` - for any NIC with more than one queue pair, and a `tap` netdev behind - `qemu-bridge-helper` whenever vhost is on; only a single-queue, non-vhost - bridge NIC keeps the historical `-netdev bridge` path with no `netd` or - libvirt dependency. +- `network_filter = "none"` installs no nwfilter binding. It does not remove + the `netd` dependency: `netd` creates the TAP for every bridge NIC either + way, and the VMM uses `-netdev tap` either way. What changes is only whether + that TAP carries a binding. - `network_filter = "libvirt"` creates the TAP and filter binding before QEMU is submitted to Supervisor, and uses QEMU `-netdev tap`. - A failed TAP or filter setup prevents QEMU from starting and rolls back all @@ -214,11 +213,9 @@ sudo dstack-vmm --config ./vmm.toml \ --netd-socket /run/dstack-dev/netd.sock ``` -User networking never asks `netd` to build an interface; the VMM still opens a -short liveness-probe connection to the netd socket on every launch and when -describing a stopped VM. Libvirt mode fails closed if `netd` -is unavailable. Bridge networking with `mode = "none"` connects only when it -needs more than one queue pair, as described below. +User networking and a caller-supplied netdev never ask `netd` to build an +interface, and a VM using only those never contacts it. Bridge and macvtap +always do, and fail closed if `netd` is unavailable. Filtered TAP netdevs follow the node's `vhost` and `queues` settings like any other TAP-backed NIC (see [network-data-plane.md](network-data-plane.md)). The @@ -227,10 +224,11 @@ whether they were written by QEMU or by a vhost worker; filtering is unaffected by the data plane choice. Enabling vhost does require the QEMU user to be able to open `/dev/vhost-net`. -`netd` also creates the TAP for unfiltered bridge NICs that ask for more than -one queue pair, because `qemu-bridge-helper` returns a single descriptor and -cannot create a `multi_queue` device. Those TAPs carry no nwfilter binding, so -a multiqueue bridge node needs `netd` even when `network_filter.mode = "none"`. +`netd` creates the TAP for unfiltered bridge NICs too. Those TAPs carry no +nwfilter binding, so a bridge node needs `netd` even when +`network_filter.mode = "none"` — see +[bridge-networking.md](bridge-networking.md) for why the host interface has a +single owner. An empty filter name is what selects that unfiltered TAP, so `mode = "libvirt"` with an empty `filter` is rejected at config load rather than quietly producing diff --git a/docs/network-data-plane.md b/docs/network-data-plane.md index 57f0134a1..c80af880f 100644 --- a/docs/network-data-plane.md +++ b/docs/network-data-plane.md @@ -64,8 +64,8 @@ refuses a request for four queue pairs should not hand out sixteen by itself. The hard ceiling from any source is 64. Without vhost the default is a single queue pair. The QEMU main loop drains -every queue on one thread, so extra queues buy little while still costing a -netd interface, more MSI-X vectors, and a changed guest device. An explicit +every queue on one thread, so extra queues buy little while still costing more +MSI-X vectors and a changed guest device. An explicit queue count is still honoured without vhost, since that combination is a deliberate request rather than a default. The two defaults travelling together also means a node that never sets `vhost` keeps building the device its VMs @@ -150,9 +150,9 @@ measured, so attestation and app identity are unaffected. Before flipping it: restarting. The VMM warns at startup when its own access fails, but it cannot refuse on that basis — QEMU need not share its credentials. -2. **Restart `netd` before or together with the VMM.** Multiqueue bridge NICs - are prepared by `netd`, and the VMM checks that `netd` echoes the queue - count it built. An older `netd` fails that check; the launch is rolled back +2. **Restart `netd` before or together with the VMM.** Every bridge and + macvtap NIC is prepared by `netd`, and the VMM checks that `netd` echoes the + queue count it built. An older `netd` fails that check; the launch is rolled back and fails with the reason in the VMM log, but the VM does not start until `netd` is upgraded. @@ -165,35 +165,22 @@ measured, so attestation and app identity are unaffected. Before flipping it: | Mode | netdev | vhost | queues > 1 | |---|---|---|---| | `user` | `user,...` | no backend | not supported | -| `bridge` | `tap,ifname=` via netd, else `tap,br=,helper=`, else `bridge,br=` | yes | yes, through netd | -| `bridge` with libvirt filtering | `tap,ifname=` | yes | yes, through netd | +| `bridge` | `tap,ifname=` via netd | yes | yes | | `macvtap` | `tap,fd=` / `tap,fds=` | yes | yes | | `custom` | operator's own string | operator's own string | no, not settable | -QEMU's `bridge` netdev accepts neither `vhost=` nor `queues=`, so enabling -vhost switches bridge mode to a `tap` netdev driven by the same setuid -`qemu-bridge-helper`. The VMM still needs no network privileges. The helper has -no compiled-in default path for the `tap` netdev, so the VMM probes the known -distribution locations; set `cvm.qemu_bridge_helper` if yours is elsewhere. If -no helper is found the NIC falls back to the non-vhost `bridge` netdev with a -warning, because a node-wide setting must not stop a node from booting VMs -that never asked for it. - -The helper returns exactly one descriptor, which is why more than one queue -pair in bridge mode is created by `netd` instead: it adds a persistent -`multi_queue` TAP that QEMU then opens once per queue. `netd` requires the -`virsh` binary to be installed even when nothing is filtered, though it does -not require a reachable `libvirtd`. That applies whether or -not libvirt filtering is on, so a bridge node needs `netd` to get the default -queue count (see [libvirt-network-filter.md](libvirt-network-filter.md)). -Without it, bridge NICs fall back to a single queue pair with a warning rather -than failing to launch; a VM that asked for a queue count explicitly still -fails, so the caller learns their request was not met. `netd` is probed by -connecting, not by looking for its socket file, because a `netd` that died -leaves the socket behind. One-shot `dstack-vmm run` has no netd lifecycle at -all and behaves like a node without it. `netd` reports back the -queue count it created, and the VMM refuses to launch on a mismatch — a `netd` -deployed separately as a root service can be older than the VMM asking it for +QEMU's `bridge` netdev accepts neither `vhost=` nor `queues=`, and the setuid +`qemu-bridge-helper` behind its `tap` netdev returns exactly one descriptor. So +bridge mode runs on a TAP that `netd` creates: persistent, `multi_queue` when +asked for, and opened once per queue by QEMU. That is true of every bridge NIC, +filtered or not, single-queue or not — see +[bridge-networking.md](bridge-networking.md) for why the host interface has one +owner. `netd` requires the `virsh` binary to be installed even when nothing is +filtered, though it does not require a reachable `libvirtd`. One-shot +`dstack-vmm run` does not manage netd interface lifecycle, so it refuses bridge +and macvtap NICs outside `--dry-run`. `netd` reports back the queue count it +created, and the VMM refuses to launch on a mismatch — a `netd` deployed +separately as a root service can be older than the VMM asking it for multiqueue, and QEMU would otherwise reject the interface from inside the per-VM launcher. @@ -253,7 +240,7 @@ multiqueue line rate with zero `swiotlb buffer is full` events. `vectors` is derived, never configured: `2N + 2`, one vector per queue direction plus config and control. One queue pair emits no `mq=on` or `vectors=` at all, leaving the guest device line byte for byte identical to the -one before this feature. The `-netdev` half does change wherever vhost is on, +one before this feature. The `-netdev` half carries `vhost=on|off` either way, since that is what selects the backend. ## Requirements diff --git a/docs/onboarding.md b/docs/onboarding.md index 0d4488bab..28b216ff2 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -149,7 +149,7 @@ The deploy command: Pass `--vcpu`, `--memory`, or `--disk` to change the app resources before you deploy. -The `--port 8080:80` mapping means `host_port:vm_port` and uses TCP on `127.0.0.1`. The full accepted forms are `vm`, `host:vm`, `proto:host:vm`, and `proto:addr:host:vm`. Use `tcp` or `udp` for `proto`. Fixed host and VM ports must be between 1 and 65535. If you omit the host port, or use `auto` or `0`, `dstack` picks a free localhost port and prints the selected mapping after deploy. +The `--port 8080:80` mapping means `host_port:vm_port` and uses TCP on `127.0.0.1`. The full accepted forms are `vm`, `host:vm`, `proto:host:vm`, and `proto:addr:host:vm`, each optionally suffixed with `@` to name which NIC the traffic enters through (a single-NIC VM never needs it). Use `tcp` or `udp` for `proto`. Fixed host and VM ports must be between 1 and 65535. If you omit the host port, or use `auto` or `0`, `dstack` picks a free localhost port and prints the selected mapping after deploy. Open the app from the host: diff --git a/dstack/crates/dstack-cli-core/src/ports.rs b/dstack/crates/dstack-cli-core/src/ports.rs index ae662442a..142be1b16 100644 --- a/dstack/crates/dstack-cli-core/src/ports.rs +++ b/dstack/crates/dstack-cli-core/src/ports.rs @@ -33,7 +33,25 @@ pub fn tcp_port_free(addr: &str, port: u16) -> bool { /// * `:` — tcp, 127.0.0.1 /// * `::` /// * `:::` +/// +/// Any of them may carry a trailing `@` naming which NIC the traffic +/// enters through, as `vmm-cli.py --port` does. Without it the VMM picks: the +/// first user-mode NIC, else the first bridge NIC. A single-NIC VM never needs +/// it. pub fn parse_port(spec: &str) -> Result { + let (spec, nic_index) = match spec.rsplit_once('@') { + // Only digits. `parse()` would take " 1" and "+1" as 1, and a NIC index + // is a position in a list the caller wrote. + Some((rest, nic)) if nic.bytes().all(|byte| byte.is_ascii_digit()) && !nic.is_empty() => ( + rest, + Some( + nic.parse::() + .with_context(|| format!("invalid NIC index in --port '{spec}'"))?, + ), + ), + Some((_, nic)) => bail!("invalid NIC index in --port '{spec}': '{nic}' is not a number"), + None => (spec, None), + }; let parts: Vec<&str> = spec.split(':').collect(); let (proto, addr, host, vm) = match parts.as_slice() { [vm] => ("tcp", "127.0.0.1", "auto", *vm), @@ -58,6 +76,7 @@ pub fn parse_port(spec: &str) -> Result { host_address: addr.to_string(), host_port, vm_port, + nic_index, }) } @@ -90,4 +109,23 @@ mod tests { assert!(parse_port("70000:80").is_err()); assert!(parse_port("8080:0").is_err()); } + + #[test] + fn a_mapping_can_name_the_nic_it_enters_through() { + assert_eq!(parse_port("8080:80").unwrap().nic_index, None); + + let pinned = parse_port("udp:0.0.0.0:7483:51820@1").unwrap(); + assert_eq!(pinned.nic_index, Some(1)); + assert_eq!(pinned.protocol, "udp"); + assert_eq!(pinned.host_address, "0.0.0.0"); + assert_eq!(pinned.host_port, 7483); + assert_eq!(pinned.vm_port, 51820); + + // Unpinned must stay unpinned rather than default to NIC 0: the VMM + // resolves it to the first user-mode NIC, which need not be the first. + assert_eq!(parse_port("8080:80@0").unwrap().nic_index, Some(0)); + for bad in ["8080:80@", "8080:80@ 1", "8080:80@+1", "8080:80@a"] { + assert!(parse_port(bad).is_err(), "{bad} must be refused"); + } + } } diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 9624c3a27..8a70141e9 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -102,7 +102,8 @@ enum Command { /// disk size in GB. #[arg(long, default_value_t = 20)] disk: u32, - /// expose a port: `vm` | `host:vm` | `proto:host:vm` | `proto:addr:host:vm` + /// expose a port: `vm` | `host:vm` | `proto:host:vm` | `proto:addr:host:vm`, + /// each optionally suffixed `@` to name the NIC it enters through /// (host omitted/`auto`/`0` ⇒ a free host port is picked). Repeatable. #[arg(long = "port", value_name = "SPEC")] ports: Vec, diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index 6599bc3cb..b32f9fea0 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -294,6 +294,9 @@ pub(crate) async fn cmd_install(mut o: InstallOpts, release_api_base_url: &str) host_address: "127.0.0.1".into(), host_port: kms_port as u32, vm_port: 8000, + // Unpinned: this deploys the node default topology, which + // is one NIC, and the VMM resolves that itself. + nic_index: None, }], ..Default::default() }; diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 528cd44b9..2d063bd36 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -35,7 +35,6 @@ use rand::seq::SliceRandom; use serde::{Deserialize, Serialize}; use serde_json::json; use sha2::{Digest, Sha256}; -use std::cell::OnceCell; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use std::net::IpAddr; use std::path::{Path, PathBuf}; @@ -46,8 +45,8 @@ use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; pub(crate) use network::{ - clamp_queues_without_netd, filters_bridge_traffic, ingress_for, needs_netd_interface, - netd_available, netd_teardown, resolve_networking, resolved_networks, settle_vhost, + filters_bridge_traffic, ingress_for, mode_carries_ingress, needs_netd_interface, netd_teardown, + resolve_networking, resolved_networks, settle_vhost, stranded_ingress, validate_resolved_network, validate_resolved_networks, }; pub use qemu::VmConfig; @@ -314,6 +313,9 @@ pub struct App { state: Arc>, /// Pull status for registry images: tag → status. pub(crate) pull_status: Arc>>, + /// One lock per VM, held across a launch or a teardown. See + /// [`App::launch_lock`]. + launch_locks: Arc>>>>, } const GUEST_AGENT_RPC_TIMEOUT: Duration = Duration::from_secs(30); @@ -344,9 +346,37 @@ impl App { })), config: Arc::new(config), pull_status: Arc::new(Mutex::new(std::collections::HashMap::new())), + launch_locks: Arc::new(Mutex::new(HashMap::new())), } } + /// Serializes everything that creates or deletes one VM's host interfaces. + /// + /// A launch reads whether QEMU is already up and then spends many awaits -- + /// a GPU reset, a whole netd conversation, building the QEMU arguments -- + /// before it launches anything. Nothing used to cover that window. Two + /// `StartVm` calls, or one racing the auto-restart timer, could both read + /// "not running", and the loser's sweep would delete the TAPs the winner's + /// QEMU was already holding open: a live VM silently loses its networking, + /// and the loser's error path then clears the winner's record of it. The + /// authoritative rejection lives in the supervisor, which is reached long + /// after the damage is done. + /// + /// A tokio mutex, because it is held across awaits. Per VM, because a slow + /// start must not stall unrelated ones. Taken by stop and by removal as + /// well as by start: those delete the same interfaces from the other side. + pub(crate) async fn launch_lock(&self, id: &str) -> tokio::sync::OwnedMutexGuard<()> { + let lock = { + let mut locks = self.launch_locks.lock().or_panic("mutex poisoned"); + // A VM that is neither starting nor stopping leaves the map holding + // the only reference, so the map stays the size of what is in + // flight rather than of every VM this process has ever touched. + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + locks.entry(id.to_string()).or_default().clone() + }; + lock.lock_owned().await + } + pub async fn load_vm( &self, work_dir: impl AsRef, @@ -424,6 +454,9 @@ impl App { vm.state.auto_restart.reset(); } } + // Everything below reads whether this VM is running and acts on the + // answer for as long as the launch takes. See [`App::launch_lock`]. + let _launch = self.launch_lock(id).await; { let state = self.lock(); if let Some(vm) = state.get(id) { @@ -549,6 +582,9 @@ impl App { if let Some(vm) = self.lock().get_mut(id) { vm.state.auto_restart.reset(); } + // Teardown deletes the same interfaces a launch creates, and derives + // their names rather than reading a record, so it must not overlap one. + let _launch = self.launch_lock(id).await; self.set_started(id, false)?; self.stop_vm_process(id).await?; let networks = self.work_dir(id)?.runtime_networks(); @@ -561,10 +597,22 @@ impl App { vm: &VmConfig, networks: &mut [Networking], ) -> Result<()> { - if !networks - .iter() - .any(|network| needs_netd_interface(network, &self.config.cvm)) - { + // Before the early return, because a mapping with nowhere to go is a + // property of the resolved topology and not of whether netd is in it. + // Deployment refuses every way of asking for one, so reaching this + // means an edit removed the NIC out from under a mapping that named it. + for mapping in stranded_ingress(&vm.manifest.port_map, networks) { + warn!( + vm_id = %vm.manifest.id, + "port mapping {} {}:{} names NIC {:?}, which this VM no longer has a backend \ + for; it will not be published", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.nic_index, + ); + } + if !networks.iter().any(needs_netd_interface) { return Ok(()); } // Whatever an earlier boot left behind, from a crash between creating @@ -602,7 +650,7 @@ impl App { // it states the requirement and lets the node's netd answer it. let mut prepared = Vec::new(); for (nic_index, network) in networks.iter_mut().enumerate() { - if !needs_netd_interface(network, &self.config.cvm) { + if !needs_netd_interface(network) { continue; } let identity = InterfaceIdentity { @@ -665,16 +713,17 @@ impl App { warn!(%cleanup_error, "failed to roll back in-flight filtered network"); } self.roll_back_prepared_networks(prepared).await; - // netd's own message is about a TAP, not about queues, so - // a caller who asked for multiqueue would not see their - // request named anywhere in the failure. + // netd's own message is about a socket or a TAP, so + // neither the NIC that asked nor what a node has to install + // to satisfy it appears anywhere in the failure. let unreachable = netd::is_unreachable(&error); + let mode = network.nic.mode.as_str(); let error = Err(error).context("failed to prepare netd-managed networking"); - return if queues > 1 && unreachable { + return if unreachable { error.with_context(|| { format!( - "interface {nic_index} asked for {queues} queue pairs, which needs \ - a netd running on this host" + "interface {nic_index} is {mode}, whose host interface only netd \ + can build; run dstack-vmm netd on this host" ) }) } else if queues > 1 { @@ -682,7 +731,7 @@ impl App { format!("interface {nic_index} asked for {queues} queue pairs") }) } else { - error + error.with_context(|| format!("interface {nic_index} is {mode}")) }; } }; @@ -722,16 +771,43 @@ impl App { Ok(()) })(); // Ports asked for and not answered for used to vanish in silence: - // no warning, and `GetInfo` still listing them. A netd that forwards - // says what it built, so nothing said means nothing forwarded. - let asked = ingress[nic_index].len(); - if asked > 0 && response.ingress.is_none() { - warn!( + // no warning, and `GetInfo` still listing them as though they + // worked. + let asked = &ingress[nic_index]; + match &response.ingress { + // A netd that forwards says what it built, so saying nothing is + // how one that does not reports it. A warning rather than a + // refusal: a VM deployed before this has been running with its + // ports dropped, and failing its launch now would turn a silent + // misconfiguration into an outage on upgrade. + None if !asked.is_empty() => warn!( vm_id = %vm.manifest.id, - ports = asked, - "netd on this node does not forward host ports, so this VM's \ - port mappings do not apply to its bridge interface" - ); + ports = asked.len(), + "netd on this node does not forward host ports, so this VM's port mappings \ + on interface {nic_index} are not published" + ), + // It answered, so hold it to the answer. A netd may refuse one + // port out of a set -- a node policy over which ports may be + // handed out is its own to state -- and the mapping that lost + // is the one worth naming. + Some(bound) => { + for request in asked { + if !bound.iter().any(|binding| { + binding.protocol == request.protocol + && binding.host_port == request.host_port + && binding.guest_port == request.guest_port + }) { + warn!( + vm_id = %vm.manifest.id, + "netd did not publish {} {}:{} on interface {nic_index}", + request.protocol, + request.host_address, + request.host_port, + ); + } + } + } + None => {} } if let Err(error) = accepted { self.roll_back_prepared_networks(prepared).await; @@ -747,45 +823,18 @@ impl App { /// down the snapshot describes a boot that is over: the node configuration /// and the VM's own manifest can both have changed since, so reporting it /// would answer a question about the past with the grammar of the present. - /// Predict instead, the same way the next launch will -- including the - /// drop to a single queue pair on a node with no netd. - /// - /// `netd_reachable` is shared across a request rather than probed here: - /// the probe is a blocking connect that netd's serialized accept loop has - /// to service, and one status query covers many VMs. - fn effective_networks( - &self, - info: &vm_info::VmInfo, - netd_reachable: &OnceCell, - ) -> Vec { + /// Predict instead, the same way the next launch will. + fn effective_networks(&self, info: &vm_info::VmInfo) -> Vec { if info.running && !info.runtime_networks.is_empty() { return info.runtime_networks.clone(); } - let available = *netd_reachable.get_or_init(|| netd_available(&self.config.netd.socket)); - self.merge_networks(&info.manifest, available).0 + self.merge_networks(&info.manifest) } - /// Launch-time view of a VM's NICs: node defaults merged in, the - /// vCPU-scaled queue count made concrete, and multiqueue dropped when this - /// node has no netd to build the interface. + /// Launch-time view of a VM's NICs: node defaults merged in and the + /// vCPU-scaled queue count made concrete. pub(crate) fn runtime_networks(&self, manifest: &Manifest) -> Vec { - let available = netd_available(&self.config.netd.socket); - let (networks, clamped, vhost_denied) = self.merge_networks(manifest, available); - if clamped > 0 { - warn!( - id = %manifest.id, - "netd is not available, so {clamped} bridge interface(s) fall back to a single \ - queue pair; run dstack-vmm netd to let queue pairs scale with vCPUs" - ); - } - if vhost_denied > 0 { - warn!( - id = %manifest.id, - "no qemu-bridge-helper found, so {vhost_denied} bridge interface(s) fall back to \ - the non-vhost bridge netdev; set cvm.qemu_bridge_helper to enable vhost" - ); - } - networks + self.merge_networks(manifest) } /// A running VM whose snapshot is missing, because a VMM that predates the @@ -802,12 +851,8 @@ impl App { /// queue pair and no vhost. Asking `runtime_networks` would apply today's /// defaults to a launch that predates them, and the guess is persisted, so /// it would keep describing that VM wrongly for the life of its boot. - /// - /// `merge_networks` rather than `runtime_networks` for the same reason: the - /// latter probes netd and warns about a multiqueue fallback, which says - /// nothing about a VM that is already up. fn inferred_runtime_networks(&self, manifest: &Manifest) -> Vec { - let mut networks = self.merge_networks(manifest, false).0; + let mut networks = self.merge_networks(manifest); for network in &mut networks { network.nic.vhost = Some(false); network.nic.queues = Some(1); @@ -822,23 +867,12 @@ impl App { networks } - /// The merge itself, without the launch-time logging, plus how many NICs - /// lost multiqueue for want of netd. - fn merge_networks( - &self, - manifest: &Manifest, - netd_reachable: bool, - ) -> (Vec, usize, usize) { - let requested = if manifest.networks.is_empty() { - vec![self.config.cvm.networking.nic.clone()] - } else { - manifest.networks.clone() - }; + /// The merge itself: node defaults applied, then the data plane settled so + /// that every later stage reads one answer instead of recomputing it. + fn merge_networks(&self, manifest: &Manifest) -> Vec { let mut resolved = resolved_networks(manifest, &self.config.cvm); - let clamped = - clamp_queues_without_netd(&requested, &mut resolved, &self.config.cvm, netd_reachable); - let vhost_denied = settle_vhost(&mut resolved, &self.config.cvm); - (resolved, clamped, vhost_denied) + settle_vhost(&mut resolved); + resolved } /// Removes interfaces netd already built for a launch that then failed. @@ -1009,8 +1043,11 @@ impl App { } let runtime_networks = self.work_dir(id)?.runtime_networks(); - if let Err(error) = self.remove_filtered_networks(id, &runtime_networks).await { - warn!(id, %error, "failed to remove filtered networking during VM removal"); + { + let _launch = self.launch_lock(id).await; + if let Err(error) = self.remove_filtered_networks(id, &runtime_networks).await { + warn!(id, %error, "failed to remove filtered networking during VM removal"); + } } // Only delete the workdir for user-initiated removal or if .removing marker exists. @@ -1391,14 +1428,11 @@ impl App { }); let total = infos.len() as u32; - // One probe for the whole page, and none at all when every VM is - // running and has its own snapshot to report. - let netd_reachable = OnceCell::new(); let vms = paginate(infos, request.page, request.page_size) .map(|vm| { let work_dir = self.work_dir(&vm.config.manifest.id)?; let info = vm.merged_info(vms.get(&vm.config.manifest.id), &work_dir); - let networks = self.effective_networks(&info, &netd_reachable); + let networks = self.effective_networks(&info); Ok(info.to_pb(&self.config.gateway, request.brief, &networks)) }) .collect::>>()?; @@ -1433,8 +1467,7 @@ impl App { }; vm_state.merged_info(proc_state.as_ref(), &self.work_dir(id)?) }; - let netd_reachable = OnceCell::new(); - let networks = self.effective_networks(&info, &netd_reachable); + let networks = self.effective_networks(&info); Ok(Some(info.to_pb(&self.config.gateway, false, &networks))) } @@ -2130,6 +2163,46 @@ mod tests { use super::mr_config::{mr_config_version, MrConfigVersion}; use super::*; + fn test_app() -> App { + use rocket::figment::providers::Format as _; + let config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + App::new(config, SupervisorClient::new("http://127.0.0.1:0")) + } + + /// The window a launch spends between reading "not running" and actually + /// starting QEMU is long -- a GPU reset, a netd conversation -- and the + /// sweep inside it deletes interfaces by deriving their names. Two entrants + /// in that window meant the loser deleting the winner's live TAPs. + #[tokio::test] + async fn one_vm_launches_at_a_time_and_the_lock_map_stays_small() { + let app = test_app(); + let held = app.launch_lock("vm-1").await; + + // A different VM is never blocked by it: a slow start must not stall + // every other launch on the node. + let other = tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-2")).await; + assert!(other.is_ok(), "an unrelated VM must not wait"); + + // The same VM is. + let same = tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-1")).await; + assert!(same.is_err(), "a second entrant must wait for the first"); + + drop(held); + drop(other); + tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-1")) + .await + .expect("the lock is released"); + + // Nothing is in flight now, so the map holds nothing either. + assert!(app.launch_locks.lock().unwrap().len() <= 1); + let _ = app.launch_lock("vm-3").await; + assert!(app.launch_locks.lock().unwrap().len() <= 2); + } + #[test] fn accepts_server_generated_ids() { validate_vm_id(&uuid::Uuid::new_v4().to_string()).unwrap(); diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index 0ee7c4e38..05fb6e42e 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -85,17 +85,22 @@ pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec bool { - match networking.nic.mode { - NetworkingMode::Macvtap => true, - NetworkingMode::Bridge => { - cfg.network_filter.mode == NetworkFilterMode::Libvirt || networking.queue_pairs() > 1 - } - NetworkingMode::User | NetworkingMode::Custom => false, - } +/// Every macvtap and every bridge NIC. The alternative was a set of conditions +/// -- libvirt filtering, multiqueue -- under which netd was consulted and +/// outside of which the VMM built the interface some other way. Each of those +/// paths had to answer the same questions again and answer them differently: +/// which netdev QEMU gets, whether vhost is really on, and, once port mappings +/// grew a NIC, where a bridge NIC's host ports go. A host interface has one +/// owner now, and `port_map` on a bridge reaches netd on every node rather than +/// only on the ones that happened to filter or to have scaled their queues. +/// +/// The cost is stated plainly: bridge and macvtap need a netd on the host. User +/// mode and a caller-supplied netdev still need nothing. +pub(crate) fn needs_netd_interface(networking: &Networking) -> bool { + matches!( + networking.nic.mode, + NetworkingMode::Macvtap | NetworkingMode::Bridge + ) } /// Whether this NIC's host interface carries a libvirt nwfilter binding. @@ -120,131 +125,23 @@ pub(crate) fn netd_teardown(networking: &Networking, cfg: &CvmConfig) -> Option< // preparation recorded the fact. Fall back to the derivation such an // entry was created by; a Remove for an interface that does not exist // is a no-op. - NetdInterface::None if needs_netd_interface(networking, cfg) => { + NetdInterface::None if needs_netd_interface(networking) => { Some(filters_bridge_traffic(networking, cfg)) } NetdInterface::None => None, } } -/// Drops a NIC back to one queue pair when multiqueue would need a netd -/// interface this node cannot provide. -/// -/// Queue pairs are a default now, not something the operator asked for, so a -/// node that has never deployed netd must keep launching bridge VMs. An -/// explicit per-VM request is left alone: the caller asked for it, and failing -/// at prepare tells them why far better than silently halving their throughput. -/// Returns how many NICs it dropped, so a launch can say so and a status -/// query, which runs the same calculation to describe a stopped VM, stays -/// silent. -pub(crate) fn clamp_queues_without_netd( - requested: &[NicNetworking], - resolved: &mut [Networking], - cfg: &CvmConfig, - netd_available: bool, -) -> usize { - if netd_available { - return 0; - } - let mut clamped = 0; - for (networking, asked) in resolved.iter_mut().zip(requested) { - // Macvtap has nothing to fall back to: netd is the only thing that can - // create the device, so clamping one would describe a VM that cannot - // start either way. - if networking.nic.mode != NetworkingMode::Bridge - || asked.queues.is_some() - || !needs_netd_interface(networking, cfg) - // Filtering needs netd whatever the queue count, so dropping this - // NIC to one queue pair would not make it launchable. It would only - // describe it as something no launch can produce, and warn about a - // fallback that is not happening. - || filters_bridge_traffic(networking, cfg) - { - continue; - } - networking.nic.queues = Some(1); - clamped += 1; - } - clamped -} - -/// Locations distributions install `qemu-bridge-helper` in. The helper is -/// setuid root and attaches an unprivileged TAP to a whitelisted bridge, which -/// is how bridge mode avoids giving the VMM `CAP_NET_ADMIN`. -const BRIDGE_HELPER_CANDIDATES: [&str; 3] = [ - "/usr/lib/qemu/qemu-bridge-helper", - "/usr/libexec/qemu-bridge-helper", - "/usr/local/libexec/qemu-bridge-helper", -]; - -/// Absolute path of `qemu-bridge-helper`, which QEMU's `tap` netdev, unlike its -/// `bridge` netdev, has no compiled-in default for. -/// -/// A configured path is passed through unchecked: the operator is naming a -/// binary for QEMU to exec, and QEMU need not see this filesystem. -pub(crate) fn find_bridge_helper<'a>( - configured: &'a str, - candidates: &[&'a str], -) -> Option<&'a str> { - let configured = configured.trim(); - if !configured.is_empty() { - return Some(configured); - } - candidates - .iter() - .copied() - .find(|candidate| Path::new(candidate).exists()) -} - -pub(crate) fn bridge_helper(cfg: &CvmConfig) -> Option<&str> { - find_bridge_helper(&cfg.qemu_bridge_helper, &BRIDGE_HELPER_CANDIDATES) -} - -/// Whether this NIC will actually run on the vhost-net data plane. -/// -/// A bridge NIC that neither needs a netd interface nor can find -/// `qemu-bridge-helper` falls back to QEMU's `bridge` netdev, which has no -/// vhost support. Both the QEMU arguments and the reported status read this, -/// so a VM is never described as using a data plane it did not get. -pub(crate) fn effective_vhost(networking: &Networking, cfg: &CvmConfig) -> bool { - if !networking.vhost_enabled() { - return false; - } - networking.nic.mode != NetworkingMode::Bridge - || needs_netd_interface(networking, cfg) - || bridge_helper(cfg).is_some() -} - -/// Makes the effective data plane concrete on a launch-time NIC list, and -/// returns how many interfaces asked for vhost and did not get it. +/// Makes the data plane concrete on a launch-time NIC list. /// /// `vhost` on a freshly resolved entry is still a *request*: `None` means -/// inherit, and a bridge NIC that cannot reach `qemu-bridge-helper` runs on the -/// non-vhost netdev whatever it asked for. Settling it once, here, is what lets -/// the QEMU arguments and the reported status read the same value -- and keeps -/// them reading it after the operator moves the helper out from under a VM that -/// is already running. -pub(crate) fn settle_vhost(networks: &mut [Networking], cfg: &CvmConfig) -> usize { - let mut denied = 0; +/// inherit from the node, which can change under a VM that is already running. +/// Settling it once, here, is what lets the QEMU arguments and the reported +/// status read the same value for the life of a boot. +pub(crate) fn settle_vhost(networks: &mut [Networking]) { for networking in networks.iter_mut() { - let effective = effective_vhost(networking, cfg); - if networking.vhost_enabled() && !effective { - denied += 1; - } - networking.nic.vhost = Some(effective); + networking.nic.vhost = Some(networking.vhost_enabled()); } - denied -} - -/// Whether netd is reachable. A netd that died leaves its socket behind, so -/// existence alone would report a node as capable and fail every launch. -/// -/// A connect and nothing more, deliberately: netd serves connections serially, -/// so anything that waits for an answer reads a *busy* netd as a missing one -/// and silently drops the VM to a single queue pair. Accepting the connection -/// is the one signal that does not depend on what netd is doing right now. -pub(crate) fn netd_available(socket: &Path) -> bool { - std::os::unix::net::UnixStream::connect(socket).is_ok() } pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { @@ -347,16 +244,45 @@ pub(crate) fn default_ingress_nic(networks: &[Networking]) -> Option { }) } +/// Whether a NIC of this mode has a mechanism to publish a host port at all. +/// +/// `hostfwd=` for user mode and netd for a bridge. Macvtap bypasses the host +/// bridge and a custom netdev is a string the VMM does not interpret, so +/// neither has anywhere to put one. Naming one of those is refused at +/// deployment rather than resolved to nothing here. +pub(crate) fn mode_carries_ingress(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::User | NetworkingMode::Bridge) +} + /// Which NIC a port mapping's traffic enters through. /// /// One mapping resolves to at most one NIC, and that NIC's backend decides the /// mechanism: `hostfwd=` for user mode, netd for a bridge. That is what keeps /// QEMU and netd from both claiming one host port. +/// +/// `None` is a mapping with nowhere to go. Deployment refuses every way of +/// asking for one, so reaching it means a manifest wrote a NIC out from under a +/// mapping that named it; the launch says so rather than dropping it in +/// silence. pub(crate) fn ingress_nic(mapping: &PortMapping, networks: &[Networking]) -> Option { mapping .nic_index .or_else(|| default_ingress_nic(networks)) - .filter(|index| *index < networks.len()) + .filter(|index| { + networks + .get(*index) + .is_some_and(|network| mode_carries_ingress(network.nic.mode)) + }) +} + +/// Names the mappings that resolve to no NIC, for a launch to warn about. +pub(crate) fn stranded_ingress<'a>( + port_map: &'a [PortMapping], + networks: &'a [Networking], +) -> impl Iterator { + port_map + .iter() + .filter(|mapping| ingress_nic(mapping, networks).is_none()) } /// The host ports one NIC carries, as netd requests. @@ -404,9 +330,9 @@ pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) #[cfg(test)] mod tests { use super::{ - clamp_queues_without_netd, default_ingress_nic, effective_vhost, ingress_for, ingress_nic, - mac_address_for_vm_index, needs_netd_interface, netd_teardown, resolve_networking, - resolved_networks, settle_vhost, validate_resolved_networks, + default_ingress_nic, ingress_for, ingress_nic, mac_address_for_vm_index, + needs_netd_interface, netd_teardown, resolve_networking, resolved_networks, settle_vhost, + stranded_ingress, validate_resolved_networks, }; use crate::app::PortMapping; use crate::config::Protocol; @@ -500,7 +426,6 @@ mod tests { let resolved = resolved_networks(&manifest_with(16, vec![]), &cvm); assert!(!resolved[0].vhost_enabled()); assert_eq!(resolved[0].queue_pairs(), 1); - assert!(!needs_netd_interface(&resolved[0], &cvm)); } } @@ -511,7 +436,6 @@ mod tests { let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); assert!(!resolved[0].vhost_enabled()); assert_eq!(resolved[0].queue_pairs(), 1); - assert!(!needs_netd_interface(&resolved[0], &cvm)); // Per-VM opt-out does the same thing. let cvm = node_config(NetworkingMode::Bridge); @@ -542,31 +466,37 @@ mod tests { assert_eq!(resolved[0].queue_pairs(), 16); } + /// One owner for a host interface, with no condition attached. What used + /// to decide this -- libvirt filtering, a scaled queue count -- decided it + /// per node, so the same VM definition got a netd-built TAP on one host and + /// a `qemu-bridge-helper` TAP on the next. #[test] - fn status_never_claims_a_data_plane_the_nic_did_not_get() { + fn every_bridge_and_macvtap_nic_is_netds_to_build() { let mut cvm = node_config(NetworkingMode::Bridge); - // No helper on this filesystem and no netd interface needed, so the - // NIC falls back to QEMU's `bridge` netdev, which has no vhost. - cvm.qemu_bridge_helper = String::new(); let mut single = cvm.networking.nic.clone(); single.queues = Some(1); + let resolved = resolved_networks(&manifest_with(8, vec![single.clone()]), &cvm); + assert!(needs_netd_interface(&resolved[0])); + + // Unfiltered, single queue, no vhost -- the shape that used to need no + // netd at all -- is netd's too. + cvm.networking.nic.vhost = Some(false); + single.vhost = Some(false); let resolved = resolved_networks(&manifest_with(8, vec![single]), &cvm); - assert!(resolved[0].vhost_enabled()); - let fell_back = !effective_vhost(&resolved[0], &cvm); - assert_eq!(fell_back, super::bridge_helper(&cvm).is_none()); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + assert!(needs_netd_interface(&resolved[0])); - // A configured helper is taken at its word, so vhost is real. - cvm.qemu_bridge_helper = "/opt/qemu-bridge-helper".into(); + let cvm = node_config(NetworkingMode::Macvtap); let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); - assert!(effective_vhost(&resolved[0], &cvm)); + assert!(needs_netd_interface(&resolved[0])); - // Multiqueue goes through netd, which needs no helper at all. - let mut mq = cvm.networking.nic.clone(); - mq.queues = Some(4); - cvm.qemu_bridge_helper = String::new(); - let resolved = resolved_networks(&manifest_with(8, vec![mq]), &cvm); - assert!(needs_netd_interface(&resolved[0], &cvm)); - assert!(effective_vhost(&resolved[0], &cvm)); + // The two backends the VMM builds itself still need nothing. + for mode in [NetworkingMode::User, NetworkingMode::Custom] { + let cvm = node_config(mode); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(!needs_netd_interface(&resolved[0])); + } } #[test] @@ -586,35 +516,6 @@ mod tests { assert_eq!(resolved[0].queue_pairs(), 1); } - #[test] - fn without_netd_a_defaulted_bridge_drops_to_one_queue_but_a_request_does_not() { - let cvm = node_config(NetworkingMode::Bridge); - - // The default is ours to lower: a node that never deployed netd must - // keep launching bridge VMs. - let requested = vec![cvm.networking.nic.clone()]; - let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); - assert_eq!(resolved[0].queue_pairs(), 8); - clamp_queues_without_netd(&requested, &mut resolved, &cvm, false); - assert_eq!(resolved[0].queue_pairs(), 1); - assert!(!needs_netd_interface(&resolved[0], &cvm)); - - // An explicit request is left alone, so prepare fails where the caller - // can see why instead of silently halving their throughput. - let mut asked = cvm.networking.nic.clone(); - asked.queues = Some(4); - let requested = vec![asked.clone()]; - let mut resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); - clamp_queues_without_netd(&requested, &mut resolved, &cvm, false); - assert_eq!(resolved[0].queue_pairs(), 4); - - // With netd present nothing is touched. - let requested = vec![cvm.networking.nic.clone()]; - let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); - clamp_queues_without_netd(&requested, &mut resolved, &cvm, true); - assert_eq!(resolved[0].queue_pairs(), 8); - } - #[test] fn validation_never_depends_on_this_process_reaching_vhost_net() { // QEMU may run under different credentials, so a NIC that asks for @@ -643,62 +544,26 @@ mod tests { #[test] fn settling_vhost_records_what_the_launch_decided() { let mut cvm = node_config(NetworkingMode::Bridge); - cvm.qemu_bridge_helper = String::new(); - let mut single = cvm.networking.nic.clone(); - single.queues = Some(1); - let manifest = manifest_with(8, vec![single]); - - // Whether this host has a helper is not the test's business; that it - // gets written down, once, is. - let helper_missing = super::bridge_helper(&cvm).is_none(); + let manifest = manifest_with(8, vec![]); let mut networks = resolved_networks(&manifest, &cvm); - assert!(networks[0].vhost_enabled(), "the request starts out on"); - assert_eq!( - settle_vhost(&mut networks, &cvm), - usize::from(helper_missing) - ); - assert_eq!(networks[0].nic.vhost, Some(!helper_missing)); - // Settling an already-settled list reports nothing new, so a relaunch - // does not warn about a fallback that already happened. - assert_eq!(settle_vhost(&mut networks, &cvm), 0); - - // A configured helper is taken at its word, so the same NIC settles on. - cvm.qemu_bridge_helper = "/opt/qemu-bridge-helper".into(); - let mut with_helper = resolved_networks(&manifest, &cvm); - assert_eq!(settle_vhost(&mut with_helper, &cvm), 0); - assert_eq!(with_helper[0].nic.vhost, Some(true)); + assert_eq!(networks[0].nic.vhost, Some(true)); + settle_vhost(&mut networks); + assert_eq!(networks[0].nic.vhost, Some(true)); - // The entry the first launch settled keeps its answer: nothing about a - // running VM is recomputed from the configuration as it stands now. - assert_eq!(networks[0].nic.vhost, Some(!helper_missing)); - } - - /// Dropping to a single queue pair is only worth doing when it makes the - /// NIC launchable. A filtered bridge needs netd whatever its queue count, - /// so clamping it would report a shape no launch can produce. - #[test] - fn a_filtered_bridge_is_not_clamped_because_it_cannot_help() { - use crate::config::NetworkFilterMode; - - let mut cvm = node_config(NetworkingMode::Bridge); - cvm.network_filter.mode = NetworkFilterMode::Libvirt; - let requested = vec![cvm.networking.nic.clone()]; - let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); - assert_eq!(resolved[0].queue_pairs(), 8); - assert_eq!( - clamp_queues_without_netd(&requested, &mut resolved, &cvm, false), - 0 - ); - assert_eq!(resolved[0].queue_pairs(), 8); + // The node turns vhost off under a VM that is already running. The + // entry the launch settled keeps its answer; the next boot gets the + // new one. + cvm.networking.nic.vhost = Some(false); + assert_eq!(networks[0].nic.vhost, Some(true)); + let mut next_boot = resolved_networks(&manifest, &cvm); + settle_vhost(&mut next_boot); + assert_eq!(next_boot[0].nic.vhost, Some(false)); - // Unfiltered, the same NIC does drop, because then it can launch. - let cvm = node_config(NetworkingMode::Bridge); - let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); - assert_eq!( - clamp_queues_without_netd(&requested, &mut resolved, &cvm, false), - 1 - ); - assert_eq!(resolved[0].queue_pairs(), 1); + // An inherited `None` becomes a decision rather than staying a request. + let mut unset = resolved_networks(&manifest, &cvm); + unset[0].nic.vhost = None; + settle_vhost(&mut unset); + assert_eq!(unset[0].nic.vhost, Some(false)); } /// Teardown has to undo what was built. Node configuration is mutable and @@ -730,11 +595,21 @@ mod tests { // asking libvirt for one would fail the removal. assert_eq!(netd_teardown(&built_unfiltered, &filtering), Some(false)); - // A NIC netd never touched stays untouched, whatever the node now says. + // A backend netd never builds stays untouched, whatever the node says. let mut untouched = unfiltered.networking.clone(); + untouched.nic.mode = NetworkingMode::User; untouched.nic.queues = Some(1); assert_eq!(netd_teardown(&untouched, &unfiltered), None); + // A bridge NIC with no record is netd's by derivation, because netd is + // now the only thing that could have built it. Teardown deletes by + // deriving names, so being wrong about a VM from an older build costs + // a sweep that finds nothing. + let mut unrecorded = unfiltered.networking.clone(); + unrecorded.nic.queues = Some(1); + assert_eq!(unrecorded.netd_interface, NetdInterface::None); + assert_eq!(netd_teardown(&unrecorded, &unfiltered), Some(false)); + // An entry persisted before preparation recorded the fact still gets // torn down by the rule that created it. let mut legacy = filtering.networking.clone(); @@ -749,8 +624,8 @@ mod tests { use crate::config::NetdInterface; let cvm = node_config(NetworkingMode::Bridge); - // Single queue and no filtering, so nothing but a stale record could - // make teardown believe netd built something. + // The node does not filter, so only a stale record could make teardown + // ask libvirt to delete a binding. let mut previous = cvm.networking.clone(); previous.nic.queues = Some(1); previous.netd_interface = NetdInterface::Filtered; @@ -758,7 +633,8 @@ mod tests { let resolved = resolve_networking(&previous.nic, &cvm, 4); assert_eq!(resolved.netd_interface, NetdInterface::None); - assert_eq!(netd_teardown(&resolved, &cvm), None); + // Back to the derivation the node's own configuration gives. + assert_eq!(netd_teardown(&resolved, &cvm), Some(false)); } #[test] @@ -821,6 +697,31 @@ mod tests { assert_eq!(ingress_nic(&mapping(443, Some(7)), &networks), None); } + /// A pin has to be checked against the backend, not just the count. + /// Naming a macvtap or custom NIC used to resolve to that index and then + /// fall out of every branch that could act on it: no `hostfwd=`, no netd + /// request, and no warning either. + #[test] + fn a_pin_to_a_backend_with_no_ingress_resolves_to_nothing() { + let networks = [ + nic(NetworkingMode::Macvtap), + nic(NetworkingMode::Custom), + nic(NetworkingMode::Bridge), + ]; + assert_eq!(ingress_nic(&mapping(443, Some(0)), &networks), None); + assert_eq!(ingress_nic(&mapping(443, Some(1)), &networks), None); + assert_eq!(ingress_nic(&mapping(443, Some(2)), &networks), Some(2)); + + // And an unpinned mapping on a VM with nowhere to put it is named, + // rather than counted as delivered. + let networks = [nic(NetworkingMode::Macvtap)]; + let port_map = [mapping(443, None), mapping(8080, Some(0))]; + let stranded: Vec<_> = stranded_ingress(&port_map, &networks) + .map(|mapping| mapping.from) + .collect(); + assert_eq!(stranded, vec![443, 8080]); + } + #[test] fn one_mapping_reaches_exactly_one_nic() { // The property that keeps QEMU and netd from both claiming a host port: diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 828230068..52fb22a24 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -10,8 +10,8 @@ use super::{ image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, network::{ - bridge_helper, ingress_nic, mac_address_for_vm_index, needs_netd_interface, - validate_resolved_networks, warn_if_vhost_net_missing, + ingress_nic, mac_address_for_vm_index, validate_resolved_networks, + warn_if_vhost_net_missing, }, pci_numa_node, round_up, GpuConfig, VmWorkDir, }; @@ -681,43 +681,25 @@ impl QemuCommandBuilder<'_> { "bridge networking: mac={mac} bridge={} vhost={vhost} queues={queues}", networking.nic.bridge ); - if needs_netd_interface(networking, self.cfg) { - // netd owns this TAP: libvirt filtering binds an - // nwfilter to it, and multiqueue needs the persistent - // IFF_MULTI_QUEUE device the bridge helper cannot make. - let tap = tap_name(&InterfaceIdentity { - instance_id: self.cfg.instance_id.clone(), - vm_id: self.vm.manifest.id.clone(), - nic_index: index, - }); - let mut netdev = format!( - "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost={}", - on_off(vhost) - ); - if queues > 1 { - netdev.push_str(&format!(",queues={queues}")); - } - netdev - } else if let Some(helper) = vhost.then(|| bridge_helper(self.cfg)).flatten() { - // QEMU's `bridge` netdev has no vhost support, but the - // same setuid helper works behind a `tap` netdev, so - // the VMM still needs no network privileges. - format!( - "tap,id={net_id},br={},helper={helper},vhost=on", - networking.nic.bridge - ) - } else if vhost { - // vhost is a node-wide setting, so a node whose helper - // sits somewhere unusual must keep booting VMs rather - // than lose every bridge NIC to a path lookup. - tracing::warn!( - "{net_id}: no qemu-bridge-helper found, falling back to the \ - non-vhost bridge netdev. set cvm.qemu_bridge_helper to enable vhost" - ); - format!("bridge,id={net_id},br={}", networking.nic.bridge) - } else { - format!("bridge,id={net_id},br={}", networking.nic.bridge) + // netd owns the TAP. It is the one component here with + // CAP_NET_ADMIN, so it is the only one that can bind an + // nwfilter, create a persistent IFF_MULTI_QUEUE device, or + // arbitrate a host port between VMM instances -- and having + // it own every bridge TAP is what keeps a VM's networking + // from depending on which of those a node happens to use. + let tap = tap_name(&InterfaceIdentity { + instance_id: self.cfg.instance_id.clone(), + vm_id: self.vm.manifest.id.clone(), + nic_index: index, + }); + let mut netdev = format!( + "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost={}", + on_off(vhost) + ); + if queues > 1 { + netdev.push_str(&format!(",queues={queues}")); } + netdev } NetworkingMode::Custom => { if !networking.netdev.contains(&format!("id={net_id}")) { @@ -1291,16 +1273,23 @@ mod tests { } #[test] - fn bridge_vhost_uses_the_bridge_helper_behind_a_tap_netdev() { - // QEMU's `bridge` netdev has no vhost support at all, so enabling the - // kernel data plane has to switch netdev types while keeping the same - // unprivileged setuid helper. + fn every_bridge_nic_gets_the_netd_tap() { + // Not only the filtered or multiqueue ones. A bridge NIC's host + // interface has one owner, so the netdev QEMU is handed does not + // change with the node's filter mode or its queue count. let (mut config, ..) = test_launch_fixture(); - config.cvm.qemu_bridge_helper = "/usr/lib/qemu/qemu-bridge-helper".into(); - let args = net_args(&config, vec![bridge_network(&config)]); - assert!(args.contains( - &"tap,id=net0,br=br0,helper=/usr/lib/qemu/qemu-bridge-helper,vhost=on".to_string() - )); + config.cvm.instance_id = "vmm-a".into(); + let mut networking = bridge_network(&config); + networking.nic.queues = Some(1); + let args = net_args(&config, vec![networking]); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=on" + ))); // A single queue pair must keep the historical device line byte for byte. assert!(args.iter().any( |arg| arg.starts_with("virtio-net-pci,netdev=net0,mac=") && !arg.contains("mq=on") @@ -1308,26 +1297,20 @@ mod tests { } #[test] - fn a_missing_bridge_helper_is_reported_rather_than_guessed() { - // Configured paths are trusted verbatim: QEMU execs them, and it need - // not share this filesystem. - assert_eq!( - crate::app::network::find_bridge_helper(" /opt/qemu-bridge-helper ", &[]), - Some("/opt/qemu-bridge-helper") - ); - assert_eq!( - crate::app::network::find_bridge_helper("", &["/nonexistent/a", "/nonexistent/b"]), - None - ); - } - - #[test] - fn disabling_vhost_restores_the_legacy_bridge_netdev() { - let (config, ..) = test_launch_fixture(); + fn disabling_vhost_keeps_the_netd_tap_and_turns_the_data_plane_off() { + let (mut config, ..) = test_launch_fixture(); + config.cvm.instance_id = "vmm-a".into(); let mut networking = bridge_network(&config); networking.nic.vhost = Some(false); let args = net_args(&config, vec![networking]); - assert!(args.contains(&"bridge,id=net0,br=br0".to_string())); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=off" + ))); } #[test] @@ -1491,19 +1474,6 @@ mod tests { network.nic.bridge = "br0".into(); network.nic.vhost = Some(false); } - let process = QemuCommandBuilder { - vm: &vm, - cfg: &config.cvm, - gpus: &GpuConfig::default(), - prepared: &prepared, - } - .build() - .unwrap(); - assert!(process - .args - .iter() - .any(|arg| arg == "bridge,id=net0,br=br0")); - config.cvm.instance_id = "vmm-a".into(); config.cvm.network_filter.mode = NetworkFilterMode::Libvirt; let process = QemuCommandBuilder { diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 4cd10762b..48af6f864 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -33,15 +33,6 @@ pub(crate) struct VmInfo { pub runtime_networks: Vec, } -fn networking_mode_name(mode: NetworkingMode) -> &'static str { - match mode { - NetworkingMode::Bridge => "bridge", - NetworkingMode::User => "user", - NetworkingMode::Custom => "custom", - NetworkingMode::Macvtap => "macvtap", - } -} - fn networking_backend_name(mode: NetworkingMode) -> &'static str { match mode { NetworkingMode::Bridge => "tap_bridge", @@ -62,7 +53,7 @@ fn interfaces_to_proto( .map(|(index, networking)| { let mac = mac_address_for_vm_index(vm_id, &networking.mac_prefix_bytes(), index); pb::NetworkInterfaceStatus { - mode: networking_mode_name(networking.nic.mode).into(), + mode: networking.nic.mode.as_str().into(), backend: networking_backend_name(networking.nic.mode).into(), mac, bridge_name: (networking.nic.mode == NetworkingMode::Bridge) @@ -106,7 +97,7 @@ pub(crate) fn networking_to_proto(networking: &NicNetworking) -> pb::NetworkingC mode: if networking.inherit_mode { String::new() } else { - networking_mode_name(networking.mode).into() + networking.mode.as_str().into() }, bridge_name: if pins_backend && networking.mode == NetworkingMode::Bridge { networking.bridge.clone() diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index d7479abc4..c7542ca84 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -357,11 +357,6 @@ pub struct CvmConfig { pub qemu_pci_hole64_size: u64, /// QEMU hotplug_off pub qemu_hotplug_off: bool, - /// Path to `qemu-bridge-helper`, used to attach an unprivileged TAP to a - /// host bridge. Empty probes the known distribution locations. - #[serde(default)] - pub qemu_bridge_helper: String, - /// TDX attestation/hash scheme policy. `legacy` keeps the existing /// digest.txt measurement path; `lite` opts into split measurement CBOR; /// `auto` selects `legacy` for @@ -772,16 +767,6 @@ impl Config { (1..=MAX_NET_QUEUES).contains(&self.cvm.max_net_queues), "cvm.max_net_queues must be between 1 and {MAX_NET_QUEUES}" ); - // The helper path is interpolated into QEMU's `-netdev` option list, - // which QEMU splits on ',' and '='. A path carrying either would not be - // passed through, it would end the option and start a bogus one, and the - // launch failure names neither this setting nor the file. Volume sources - // are rejected for the same reason. - anyhow::ensure!( - !self.cvm.qemu_bridge_helper.contains([',', '=']), - "cvm.qemu_bridge_helper must not contain ',' or '=': {}", - self.cvm.qemu_bridge_helper - ); anyhow::ensure!( !self .cvm @@ -978,6 +963,19 @@ pub enum NetworkingMode { Macvtap, } +impl NetworkingMode { + /// The name this mode is written as in `vmm.toml`, in the RPC, and in + /// anything an operator reads. + pub fn as_str(self) -> &'static str { + match self { + NetworkingMode::User => "user", + NetworkingMode::Bridge => "bridge", + NetworkingMode::Custom => "custom", + NetworkingMode::Macvtap => "macvtap", + } + } +} + /// What a single NIC pins: the fields a deployment may name, a VM's manifest /// stores, and `GetInfo` reports back. /// @@ -1694,26 +1692,6 @@ mod networking_shape_tests { assert!(stored.get("net").is_none()); } - /// The path is interpolated into QEMU's `-netdev` option list, which QEMU - /// splits on ',' and '='. A path carrying either would end the option and - /// start a bogus one, and the launch failure names neither the setting nor - /// the file. - #[test] - fn a_bridge_helper_path_cannot_end_the_qemu_option_it_sits_in() { - use rocket::figment::providers::Format as _; - let mut config: Config = rocket::figment::Figment::from( - rocket::figment::providers::Toml::string(DEFAULT_CONFIG), - ) - .extract() - .unwrap(); - config.cvm.qemu_bridge_helper = "/opt/qemu,helper".into(); - let error = config.validate().unwrap_err(); - assert!(error.to_string().contains("qemu_bridge_helper"), "{error}"); - - config.cvm.qemu_bridge_helper = "/usr/libexec/qemu-bridge-helper".into(); - config.validate().unwrap(); - } - /// The TOML section still deserializes through the flatten. #[test] fn the_node_section_still_parses_from_toml() { diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 158b49c10..180c0102b 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -25,8 +25,9 @@ use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; use crate::app::{ - needs_swtpm, resolve_networking, validate_resolved_network, validate_resolved_networks, App, - AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, + mode_carries_ingress, needs_swtpm, resolve_networking, validate_resolved_network, + validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, + VmWorkDir, }; use crate::config::{CvmConfig, Networking, NetworkingMode, NicNetworking}; @@ -168,23 +169,57 @@ fn port_mappings_conflict(left: &PortMapping, right: &PortMapping) -> bool { || right.address.is_unspecified()) } -/// Rejects a mapping pinned to a NIC the VM does not have. +/// The backend each of a VM's NICs resolves to, with an empty list standing for +/// the node default's single NIC. +fn resolved_nic_modes( + networks: &[NicNetworking], + cvm_config: &CvmConfig, + vcpu: u32, +) -> Vec { + let node_default = [cvm_config.networking.nic.clone()]; + let requested = if networks.is_empty() { + &node_default[..] + } else { + networks + }; + requested + .iter() + .map(|networking| resolve_networking(networking, cvm_config, vcpu).nic.mode) + .collect() +} + +/// Rejects a mapping pinned to a NIC that cannot carry it. /// -/// Range only. Whether the named NIC's backend can carry a host port is -/// resolved at launch, where the node configuration that decides it is the one -/// in force -- and where an existing VM gets a warning rather than a refusal. -fn validate_port_mapping_nics(mappings: &[PortMapping], nic_count: usize) -> Result<()> { +/// Both ways of getting that wrong are refused here, at deployment, where the +/// caller is present to be told: a NIC the VM does not have, and one whose +/// backend has no mechanism to publish a host port. Macvtap bypasses the host +/// bridge and a custom netdev is a string the VMM does not interpret, so a +/// mapping pinned to either used to resolve to that NIC and then fall out of +/// every branch that could act on it -- no `hostfwd=`, no netd request, and no +/// warning either. +fn validate_port_mapping_nics(mappings: &[PortMapping], modes: &[NetworkingMode]) -> Result<()> { + let nic_count = modes.len(); for mapping in mappings { let Some(index) = mapping.nic_index else { continue; }; - if index >= nic_count { + let Some(mode) = modes.get(index) else { bail!( "port mapping {} {}:{} names NIC {index}, but this VM has {nic_count}", mapping.protocol.as_str(), mapping.address, mapping.from ); + }; + if !mode_carries_ingress(*mode) { + bail!( + "port mapping {} {}:{} names NIC {index}, which is {} and cannot publish a host \ + port; use a user-mode or bridge NIC", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mode.as_str(), + ); } } Ok(()) @@ -244,8 +279,10 @@ pub fn create_manifest_from_vm_config( .collect::>>()?; validate_unique_port_mappings(&port_map)?; let networks = networks_from_vm_config(&request, cvm_config)?; - // An empty list inherits the node default, which is one NIC. - validate_port_mapping_nics(&port_map, networks.len().max(1))?; + validate_port_mapping_nics( + &port_map, + &resolved_nic_modes(&networks, cvm_config, request.vcpu), + )?; let app_id = match &request.app_id { Some(id) => id.strip_prefix("0x").unwrap_or(id).to_lowercase(), @@ -979,6 +1016,9 @@ impl VmmRpc for RpcHandler { .is_some_and(|info| info.state.status.is_running()); if !is_running { let runtime_networks = vm_work_dir.runtime_networks(); + // Teardown deletes interfaces by deriving their names, so it + // must not overlap a launch of the same VM. + let _launch = self.app.launch_lock(&request.id).await; self.app .remove_filtered_networks(&request.id, &runtime_networks) .await @@ -989,7 +1029,10 @@ impl VmmRpc for RpcHandler { } // After both, since either half can move and the other still has to // agree with it. - validate_port_mapping_nics(&manifest.port_map, manifest.networks.len().max(1))?; + validate_port_mapping_nics( + &manifest.port_map, + &resolved_nic_modes(&manifest.networks, &self.app.config.cvm, manifest.vcpu), + )?; let compose_file = fs::read_to_string(vm_work_dir.app_compose_path()) .context("failed to read app compose for swtpm decision")?; manifest.swtpm = needs_swtpm( @@ -1374,6 +1417,64 @@ mod tests { } } + fn pinned(nic_index: Option) -> PortMapping { + PortMapping { + address: "0.0.0.0".parse().unwrap(), + protocol: crate::config::Protocol::Tcp, + from: 443, + to: 443, + nic_index, + } + } + + /// Both ways of naming a NIC that cannot publish a port are refused where + /// the caller is present to be told. A pin to macvtap or to a custom + /// netdev used to resolve to that NIC and then fall out of every branch + /// that could act on it, so the port simply never appeared. + #[test] + fn a_pin_is_checked_against_the_backend_and_not_only_the_count() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.parent = "eth0".into(); + cvm.networking.nic.bridge = "br0".into(); + + let bridge_then_macvtap = vec![ + NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + ..NicNetworking::default() + }, + NicNetworking { + mode: NetworkingMode::Macvtap, + parent: "eth0".into(), + ..NicNetworking::default() + }, + ]; + let modes = resolved_nic_modes(&bridge_then_macvtap, &cvm, 2); + assert_eq!(modes, vec![NetworkingMode::Bridge, NetworkingMode::Macvtap]); + validate_port_mapping_nics(&[pinned(Some(0))], &modes).unwrap(); + + let error = validate_port_mapping_nics(&[pinned(Some(1))], &modes).unwrap_err(); + assert!(error.to_string().contains("macvtap"), "{error}"); + assert!( + error.to_string().contains("cannot publish a host port"), + "{error}" + ); + + let error = validate_port_mapping_nics(&[pinned(Some(2))], &modes).unwrap_err(); + assert!(error.to_string().contains("this VM has 2"), "{error}"); + + // An unpinned mapping is never refused here: where it lands is + // resolved at launch, from the topology in force then. + validate_port_mapping_nics(&[pinned(None)], &modes).unwrap(); + + // An empty list is the node default's one NIC, resolved the same way a + // launch would resolve it rather than assumed to be user mode. + cvm.networking.nic.mode = NetworkingMode::Macvtap; + let modes = resolved_nic_modes(&[], &cvm, 2); + assert_eq!(modes, vec![NetworkingMode::Macvtap]); + assert!(validate_port_mapping_nics(&[pinned(Some(0))], &modes).is_err()); + } + #[test] fn create_without_networking_persists_following_default() { let manifest = @@ -1664,6 +1765,9 @@ mod tests { .expect("a named backend is an override"); } + /// Deliberately restated rather than calling `NetworkingMode::as_str`: the + /// test below checks that what `GetInfo` reports is accepted back, and a + /// helper that shares the production mapping could only ever agree with it. fn networking_mode_name_for_test(mode: NetworkingMode) -> &'static str { match mode { NetworkingMode::Bridge => "bridge", diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 024dc7db0..1415eadae 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -5,6 +5,7 @@ //! Small privileged broker for TAP creation and libvirt nwfilter bindings. use std::{ + collections::HashSet, fs::{File, OpenOptions, Permissions}, io::Write as _, os::{ @@ -110,7 +111,10 @@ pub struct IngressRequest { /// differ only here. #[serde(default)] pub host_address: String, - /// Host port. Zero asks netd to choose one. + /// Host port, as the deployment named it. There is no "pick one for me": + /// the caller reports this number back through `GetInfo` and a client + /// connects to it, so a netd-chosen port would have to travel back through + /// both before it meant anything. pub host_port: u16, pub guest_port: u16, } @@ -206,7 +210,8 @@ struct Response { /// What netd built, echoed back so the caller can verify it matches the /// request before handing the interface to QEMU. struct Prepared { - tap: String, + /// The interface this names, absent for an operation that names none. + tap: Option, device: Option, queues: Option, ingress: Option>, @@ -216,7 +221,7 @@ struct Prepared { impl Prepared { fn tap(tap: String) -> Self { Self { - tap, + tap: Some(tap), device: None, queues: None, ingress: None, @@ -227,7 +232,7 @@ impl Prepared { /// A sweep names no single interface, so it reports how many it deleted. fn removed(removed: usize) -> Self { Self { - tap: String::new(), + tap: None, device: None, queues: None, ingress: None, @@ -427,7 +432,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul let response = match outcome { Ok(prepared) => Response { ok: true, - tap: Some(prepared.tap), + tap: prepared.tap, device: prepared.device, queues: prepared.queues, ingress: prepared.ingress, @@ -582,7 +587,7 @@ fn prepare_macvtap( Ok(device) => { info!(%tap, %parent, %mode, %device, %queues, "prepared macvtap"); Ok(Prepared { - tap, + tap: Some(tap), device: Some(device), queues: Some(queues), ingress: None, @@ -665,7 +670,7 @@ fn prepare_bridge( } info!(%tap, bridge = %request.bridge, %filtered, %queues, "prepared TAP"); Ok(Prepared { - tap, + tap: Some(tap), device: None, queues: Some(queues), // This netd builds interfaces; it is not the host's forwarder. Saying @@ -697,6 +702,14 @@ enum BindingCleanup { /// is enumerable: 256 names, each a `stat` that usually misses. Cleanup is /// best-effort about bindings -- nothing is about to take these names, and a /// node running unfiltered TAPs need not have libvirtd at all. +/// +/// A name with no interface is not skipped. An nwfilter binding outlives the +/// TAP it was bound to, so the one state teardown must not leave behind is +/// exactly the one a `/sys/class/net` check cannot see: the per-name Remove +/// this replaced deleted the binding unconditionally, and a sweep that reaches +/// less than the thing it replaced is not a sweep. Those names are decided +/// against a single listing, because the whole point of enumerating a bounded +/// space is that deciding one name stays cheap. fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Result { let identity = InterfaceIdentity { instance_id: instance_id.to_string(), @@ -704,6 +717,7 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res nic_index: 0, }; validate_identity(&identity)?; + let bindings = existing_bindings(libvirt_uri); let mut removed = 0; let mut first_error = None; for nic_index in 0..=MAX_NIC_INDEX { @@ -711,7 +725,19 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res nic_index, ..identity.clone() }); - if !Path::new("/sys/class/net").join(&tap).exists() { + let present = Path::new("/sys/class/net").join(&tap).exists(); + if !present { + if bindings + .as_ref() + .is_some_and(|bindings| bindings.contains(&tap)) + { + if let Err(error) = delete_binding(libvirt_uri, &tap) { + warn!(%tap, %error, "failed to remove orphaned nwfilter binding"); + first_error.get_or_insert(error); + } else { + info!(%tap, %vm_id, "removed orphaned nwfilter binding"); + } + } continue; } match remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort) { @@ -736,6 +762,14 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Result<()> { let macvtap = is_macvtap(tap); if Path::new("/sys/class/net").join(tap).exists() { + // The name is 48 bits of SHA-256, so a collision is not the worry. A + // caller asserting an identity that happens to derive to some + // pre-existing device is: netd runs as root and `ip link delete` does + // not ask what it is deleting. netd creates exactly two kinds of + // device, and the kernel publishes an attribute unique to each. + if !macvtap && !is_tuntap(tap) { + bail!("refusing to delete {tap}: it is neither a tun/tap nor a macvtap device"); + } let _ = ip(&["link", "set", "dev", tap, "down"]); } // A macvtap interface never carries a binding. Anything else might: this @@ -761,6 +795,16 @@ fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Re Ok(()) } +/// Whether this is a tun/tap device. `tun_flags` is published by the tun +/// driver and by nothing else, so its presence is the kernel's own answer -- +/// as `macvtap/` is for the other kind of device netd creates. +fn is_tuntap(interface: &str) -> bool { + Path::new("/sys/class/net") + .join(interface) + .join("tun_flags") + .exists() +} + fn is_macvtap(interface: &str) -> bool { Path::new("/sys/class/net") .join(interface) @@ -943,13 +987,33 @@ fn ip(args: &[&str]) -> Result<()> { } fn virsh(uri: &str, args: &[&str], stdin: Option<&[u8]>) -> Result<()> { + virsh_output(uri, args, stdin).map(|_| ()) +} + +fn virsh_output(uri: &str, args: &[&str], stdin: Option<&[u8]>) -> Result { let mut full_args = vec!["--connect", uri]; full_args.extend_from_slice(args); - run_command(VIRSH_PATH, &full_args, stdin) + run_command_with_timeout(VIRSH_PATH, &full_args, stdin, COMMAND_TIMEOUT) +} + +/// Every nwfilter binding libvirt currently holds, by interface name. +/// +/// One call, so that a sweep can decide 256 names against a set instead of +/// asking libvirt 256 times. `None` means libvirt could not be asked at all, +/// which on a node running unfiltered TAPs is the normal state -- `virsh` must +/// be installed for netd to start, but `libvirtd` need not be running. +fn existing_bindings(uri: &str) -> Option> { + match virsh_output(uri, &["nwfilter-binding-list", "--name"], None) { + Ok(output) => Some(output.split_whitespace().map(str::to_string).collect()), + Err(error) => { + debug!("could not list nwfilter bindings: {error:#}"); + None + } + } } fn run_command(program: &str, args: &[&str], stdin: Option<&[u8]>) -> Result<()> { - run_command_with_timeout(program, args, stdin, COMMAND_TIMEOUT) + run_command_with_timeout(program, args, stdin, COMMAND_TIMEOUT).map(|_| ()) } fn run_command_with_timeout( @@ -957,7 +1021,7 @@ fn run_command_with_timeout( args: &[&str], stdin: Option<&[u8]>, command_timeout: Duration, -) -> Result<()> { +) -> Result { let mut child = Command::new(program) .args(args) .stdin(if stdin.is_some() { @@ -989,7 +1053,7 @@ fn run_command_with_timeout( let error = String::from_utf8_lossy(&output.stderr); bail!("{} failed: {}", Path::new(program).display(), error.trim()); } - Ok(()) + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } fn require_executable(path: &str) -> Result<()> { @@ -1515,4 +1579,37 @@ mod tests { identity.nic_index = MAX_NIC_INDEX + 1; assert!(validate_identity(&identity).is_err()); } + + /// A sweep names no single interface, so its answer must not carry an + /// empty one. `request()` reads a blank `tap` as a malformed response, and + /// a third-party netd copying this shape would have to send a field that + /// means nothing. + #[test] + fn a_sweep_reports_a_count_and_no_interface() { + let response = Response { + ok: true, + tap: Prepared::removed(3).tap, + device: None, + queues: None, + ingress: None, + removed: Some(3), + error: None, + }; + let value = serde_json::to_value(&response).unwrap(); + assert_eq!(value["removed"], 3); + assert!(value.get("tap").is_none()); + + // A prepare still names one, because the caller hands that name to QEMU. + let value = serde_json::to_value(Response { + ok: true, + tap: Prepared::tap("dtabc".into()).tap, + device: None, + queues: None, + ingress: None, + removed: None, + error: None, + }) + .unwrap(); + assert_eq!(value["tap"], "dtabc"); + } } diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index d7500b008..62c4bd78b 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -3,9 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 use crate::app::{ - clamp_queues_without_netd, make_sys_config, needs_netd_interface, resolved_networks, - settle_vhost, simulator_config_for_manifest, sync_tee_simulator_config, Image, VmConfig, - VmWorkDir, + make_sys_config, needs_netd_interface, resolved_networks, settle_vhost, + simulator_config_for_manifest, sync_tee_simulator_config, Image, VmConfig, VmWorkDir, }; use crate::config::Config; use crate::main_service; @@ -280,42 +279,19 @@ Compose file content (first 200 chars): gateway_enabled: app_compose.gateway_enabled(), }; - // One-shot has no netd lifecycle, so a bridge NIC that only wanted the - // vCPU-scaled default drops to a single queue here exactly as it would on a - // server without netd. Anything still needing an interface was asked for - // explicitly, and is refused rather than silently downgraded. - let requested = if manifest.networks.is_empty() { - vec![config.cvm.networking.nic.clone()] - } else { - manifest.networks.clone() - }; let mut runtime_networks = resolved_networks(&manifest, &config.cvm); - let clamped = clamp_queues_without_netd(&requested, &mut runtime_networks, &config.cvm, false); - // The server settles vhost after clamping, because clamping changes whether - // a NIC needs netd and that changes which netdev it gets. Skipping it here - // left `vhost_enabled()` reading as a request rather than a decision, so - // the launch warned about a `/dev/vhost-net` the netdev it then built does - // not open. - let vhost_denied = settle_vhost(&mut runtime_networks, &config.cvm); - if vhost_denied > 0 { - tracing::warn!( - "no qemu-bridge-helper found, so {vhost_denied} bridge interface(s) fall back to the \ - non-vhost bridge netdev; set cvm.qemu_bridge_helper to enable vhost" - ); - } - if clamped > 0 { - tracing::warn!( - "one-shot execution has no netd, so {clamped} bridge interface(s) fall back to a \ - single queue pair; run the VMM server to let queue pairs scale with vCPUs" - ); - } - if !dry_run - && runtime_networks - .iter() - .any(|network| needs_netd_interface(network, &config.cvm)) - { + // Settle the data plane before anything reads it, so `vhost_enabled()` is a + // decision rather than a request and the launch does not warn about a + // `/dev/vhost-net` the netdev it then builds never opens. + settle_vhost(&mut runtime_networks); + // Bridge and macvtap host interfaces belong to netd, whose lifecycle + // one-shot does not manage. Refusing is the honest answer: the alternative + // was to quietly build a different interface here than the server would, + // and then report the VM as if it had the one it asked for. + if !dry_run && runtime_networks.iter().any(needs_netd_interface) { anyhow::bail!( - "one-shot execution does not manage netd interface lifecycle; run the VMM server directly or use --dry-run" + "one-shot execution does not manage netd interface lifecycle, which bridge and \ + macvtap networking need; run the VMM server directly or use --dry-run" ); } diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index faebc9d0f..f4e1a7b52 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -330,12 +330,11 @@ def parse_port_mapping(port_str: str) -> Dict: nic_index = None if "@" in port_str: port_str, _, nic = port_str.rpartition("@") - try: - nic_index = int(nic) - except ValueError: - raise argparse.ArgumentTypeError(f"Invalid NIC index: {nic}") - if nic_index < 0: + # `int()` alone would take "1_0" as 10, " 1" as 1, and "+1" as 1. A NIC + # index is a position in a list the user wrote, so only digits are it. + if not (nic.isascii() and nic.isdigit()): raise argparse.ArgumentTypeError(f"Invalid NIC index: {nic}") + nic_index = int(nic) parts = port_str.split(":") if len(parts) == 3: mapping = { diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index 27adcaecd..b07b36255 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -293,7 +293,10 @@ const CreateVmDialogComponent = {
- +
diff --git a/dstack/vmm/ui/src/components/PortMappingEditor.ts b/dstack/vmm/ui/src/components/PortMappingEditor.ts index 0668bd952..88f0c945c 100644 --- a/dstack/vmm/ui/src/components/PortMappingEditor.ts +++ b/dstack/vmm/ui/src/components/PortMappingEditor.ts @@ -8,6 +8,10 @@ type PortEntry = { host_port: number | null; vm_port: number | null; custom_ip?: string; // User-entered IP for custom mode + // Which NIC the traffic enters through. Blank lets the VMM pick: the first + // user-mode NIC, else the first bridge NIC. A single-NIC VM never needs it, + // which is why the field only appears once a VM has more than one. + nic_index?: number | null; }; // ... keep your types as-is ... @@ -21,6 +25,9 @@ const PortMappingEditorComponent = { name: 'PortMappingEditor', props: { ports: { type: Array, required: true }, + // How many NICs the VM has. One NIC has nothing to choose between, so the + // column stays hidden rather than offering a pin that can only be 0. + nicCount: { type: Number, default: 1 }, }, // normalize on initial load @@ -63,6 +70,15 @@ const PortMappingEditorComponent = { + @@ -95,6 +111,7 @@ const PortMappingEditorComponent = { custom_ip: '', host_port: null, vm_port: null, + nic_index: null, }); }, diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index 70e9db9df..79ed62984 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -107,6 +107,13 @@ type PortFormEntry = { host_address?: string; host_port?: number | null; vm_port?: number | null; + /** + * Which NIC this mapping's traffic enters through. Unset lets the VMM pick. + * Carried through edits unchanged: `GetInfo` reports it and this form sends + * the whole list back, so dropping it here would silently unpin a mapping + * whenever anyone touched an unrelated field. + */ + nic_index?: number | null; }; type NetworkFormEntry = { @@ -435,6 +442,7 @@ fi host_address: port.host_address || '127.0.0.1', host_port: typeof port.host_port === 'number' ? port.host_port : null, vm_port: typeof port.vm_port === 'number' ? port.vm_port : null, + nic_index: typeof port.nic_index === 'number' ? port.nic_index : null, })); const normalizePorts = (ports: PortFormEntry[] = []): VmmTypes.IPortMapping[] => @@ -445,11 +453,20 @@ fi port.host_port === null || port.host_port === undefined ? Number.NaN : Number(port.host_port); const vmPort = port.vm_port === null || port.vm_port === undefined ? Number.NaN : Number(port.vm_port); + // An unpinned mapping must stay unpinned rather than become NIC 0: + // the VMM's own default is the first user-mode NIC, not the first NIC. + const nicIndex = + port.nic_index === null || port.nic_index === undefined + ? undefined + : Number(port.nic_index); return { protocol, host_address: (port.host_address || '127.0.0.1').trim() || '127.0.0.1', host_port: hostPort, vm_port: vmPort, + ...(Number.isInteger(nicIndex) && (nicIndex as number) >= 0 + ? { nic_index: nicIndex } + : {}), }; }) .filter( diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 3a83d5016..6ff5a9e0d 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -58,11 +58,7 @@ allowed_macvtap_parents = [] # vhost on, queue pairs otherwise default to the VM's vCPU count, capped at # 16 (without vhost the default is a single queue pair); raising this # above 16 widens what a caller may request without moving that default, and -# lowering it below 16 lowers the default too. Bridge mode needs netd for -# anything above 1, because qemu-bridge-helper cannot create a multiqueue TAP. -# Without netd an unfiltered bridge NIC that took the default drops to one -# queue; one that asked for a count keeps it and fails to launch instead, so -# the caller learns their request was not met. +# lowering it below 16 lowers the default too. max_net_queues = 16 use_mrconfigid = true @@ -72,10 +68,6 @@ use_mrconfigid = true #qemu_version = "" qemu_pci_hole64_size = 0 qemu_hotplug_off = false -# Path to qemu-bridge-helper, needed by vhost bridge networking because QEMU's -# `tap` netdev, unlike its `bridge` netdev, has no compiled-in default. Empty -# probes the known distribution locations. -#qemu_bridge_helper = "/usr/lib/qemu/qemu-bridge-helper" # TDX attestation/hash scheme policy: # - "legacy": digest.txt + legacy verifier # - "lite": digest.txt + measurement.tdx.cbor + no-QEMU verifier @@ -144,16 +136,17 @@ restrict = false # bridge = "virbr0" # Optional filtering for bridge interfaces only. It does not apply to macvtap. -# "none" installs no nwfilter binding. It does not by itself remove the netd -# dependency: netd also builds the multiqueue TAP that qemu-bridge-helper -# cannot create, so a bridge node without netd is limited to one queue pair. +# "none" installs no nwfilter binding. It does not remove the netd dependency: +# netd builds every bridge and macvtap host interface whatever this says. [cvm.network_filter] mode = "none" filter = "clean-traffic" parameters = {} -# Shared privileged networking service. Used for macvtap NICs, for libvirt -# filtering, and for multiqueue bridge NICs. Socket filesystem permissions +# Shared privileged networking service. Required by bridge and macvtap +# networking: it builds every host interface those modes use, binds their +# nwfilters, and arbitrates host ports between VMM instances. User mode and a +# caller-supplied netdev need nothing from it. Socket filesystem permissions # authorize clients. [netd] socket = "/run/dstack/netd.sock" From 1ff1cf9123dadf6b6f9cc80a44231e3b2ca1a083 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 02:07:16 -0700 Subject: [PATCH 04/34] fix(vmm): ask netd what it can do instead of guessing from failures `remove_all` is an operation, and an operation a netd does not have answers the same way one that failed does: `ok: false` with a message. The VMM read that as a failed sweep, so against any netd older than this branch a stop returned an error *and* left every interface behind -- strictly worse than the per-NIC removal it replaced. `removed` was shaped to carry the other half of the signal and then discarded with `unwrap_or_default`, which turns "this netd does not sweep" into "this VM had nothing to remove": the exact conflation `queues` and `ingress` are shaped to avoid. So ask. `hello` answers a capability set, is served before the operation lock so a probe never waits behind a collection, and is bounded well below the request timeout because every teardown asks it. A netd that predates it answers an error, which is still an answer: reached, and old. The three states a caller has to act on -- absent, old, capable -- are distinguishable here and nowhere else. Teardown is then unconditional and non-fatal, which is one decision made twice. It was gated on `needs_netd_interface`, which reads the VM's *current* backend: a VM whose NIC was a bridge when its TAP was built and is user-mode now skipped the release for interfaces that exist, and neither its stop nor its next launch would ever reach them again. And it returned an error a stop propagated, so a netd outage became a fleet that could not be stopped. Nothing is lost by not failing: a launch releases before it prepares, so the next one is self-healing. `is_unreachable` never returned true. The marker is attached with `context`, which makes it the context *of* a chain link rather than a link, so the walk over `chain()` could not see it -- every "an unreachable netd is not a failure" branch in this crate was dead. `downcast_ref` sees it. And a fake netd to talk to. The VMM's side of this protocol had no test at all, because every path looked like it needed a privileged daemon; it needs something that answers on a socket. --- dstack/vmm/src/app.rs | 290 +++++++++++++----- dstack/vmm/src/main_service.rs | 5 +- dstack/vmm/src/netd.rs | 541 +++++++++++++++++++++++++++++---- 3 files changed, 704 insertions(+), 132 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 2d063bd36..819da4fe3 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -316,8 +316,27 @@ pub struct App { /// One lock per VM, held across a launch or a teardown. See /// [`App::launch_lock`]. launch_locks: Arc>>>>, + /// What this node's netd last said it can do, and when. See + /// [`App::netd_capabilities`]. + netd_probe: Arc>>, } +/// How long a netd capability answer is reused. Short, because netd is +/// upgraded and restarted under a running VMM, and a VMM that cached "this one +/// cannot sweep" across the upgrade that gave it the ability would keep +/// falling back for as long as it stayed up. +const NETD_PROBE_TTL: Duration = Duration::from_secs(30); + +/// How far past a VM's recorded NIC count a teardown reaches when netd is too +/// old to sweep by identity. +/// +/// The record is what a legacy netd leaves us: it cannot derive the space +/// itself, and asking it for all 256 possible indices would be 256 round trips +/// on every stop. Eight covers a lost NIC or two on any topology anyone +/// deploys, each miss is one cheap no-op, and what it does not reach is +/// collected the next time the node runs a netd that can sweep. +const LEGACY_TEARDOWN_SPAN: usize = 8; + const GUEST_AGENT_RPC_TIMEOUT: Duration = Duration::from_secs(30); impl App { @@ -347,6 +366,7 @@ impl App { config: Arc::new(config), pull_status: Arc::new(Mutex::new(std::collections::HashMap::new())), launch_locks: Arc::new(Mutex::new(HashMap::new())), + netd_probe: Arc::new(Mutex::new(None)), } } @@ -525,15 +545,13 @@ impl App { ) { Ok(processes) => processes, Err(error) => { - let _ = self - .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) + self.release_vm_interfaces(&vm_config.manifest.id, &runtime_networks) .await; return Err(error); } }; if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) { - let _ = self - .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) + self.release_vm_interfaces(&vm_config.manifest.id, &runtime_networks) .await; return Err(error); } @@ -544,12 +562,8 @@ impl App { } for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { - if let Err(cleanup_error) = self - .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) - .await - { - warn!(id, %cleanup_error, "failed to roll back filtered networking"); - } + self.release_vm_interfaces(&vm_config.manifest.id, &runtime_networks) + .await; if let Err(clear_err) = work_dir.clear_runtime_networks() { warn!( id, @@ -588,7 +602,11 @@ impl App { self.set_started(id, false)?; self.stop_vm_process(id).await?; let networks = self.work_dir(id)?.runtime_networks(); - self.remove_filtered_networks(id, &networks).await?; + // Not fallible: a VM that has been asked to stop is stopped whether or + // not netd could be reached, and what is left behind is collected by + // the next launch or by reconciliation. See + // [`App::release_vm_interfaces`]. + self.release_vm_interfaces(id, &networks).await; Ok(()) } @@ -612,24 +630,21 @@ impl App { mapping.nic_index, ); } + // Whatever an earlier boot left behind: a crash between creating an + // interface and recording it, a NIC this VM no longer has, or a whole + // backend it no longer uses. Prepare replaces the names it is about to + // use, but only those, so an index nothing will claim again is only + // reachable from here. + // + // Before the early return, and not inside it. A VM that has moved from + // a bridge to user-mode networking needs this release precisely because + // it no longer wants an interface, and gating it on wanting one is how + // the interfaces it left behind would become unreachable to every + // later launch. + self.release_vm_interfaces(&vm.manifest.id, networks).await; if !networks.iter().any(needs_netd_interface) { return Ok(()); } - // Whatever an earlier boot left behind, from a crash between creating - // an interface and recording it or from a NIC this VM no longer has. - // Prepare replaces the names it is about to use, but only those; an - // index nothing will claim again is only reachable from here. - if let Err(error) = netd::remove_all( - &self.config.netd.socket, - &self.config.cvm.instance_id, - &vm.manifest.id, - ) - .await - { - if !netd::is_unreachable(&error) { - warn!(vm_id = %vm.manifest.id, %error, "failed to sweep stale netd interfaces"); - } - } // Resolved before the loop borrows `networks` mutably, and once rather // than per NIC, so both the request and the warning below read the same // answer. @@ -889,50 +904,109 @@ impl App { } } - /// Deletes every host interface netd holds for this VM. + /// What this node's netd can do, cached for [`NETD_PROBE_TTL`]. + /// + /// Asked rather than inferred, and asked once per window rather than per + /// VM: every launch, teardown and reconciliation needs the answer, and the + /// probe is a connection netd's serialized accept loop has to service. + /// An unreachable answer is not cached -- a failed connect costs nothing, + /// and holding on to it would keep a VMM blind to the netd an operator + /// just started. + pub(crate) async fn netd_capabilities(&self) -> netd::Reachability { + if let Some((_, reachability)) = self + .netd_probe + .lock() + .or_panic("mutex poisoned") + .as_ref() + .filter(|(asked, _)| asked.elapsed() < NETD_PROBE_TTL) + { + return reachability.clone(); + } + let reachability = netd::probe(&self.config.netd.socket).await; + if reachability.is_reachable() { + *self.netd_probe.lock().or_panic("mutex poisoned") = + Some((std::time::Instant::now(), reachability.clone())); + } + reachability + } + + /// Releases every host interface netd holds for this VM. /// - /// A sweep rather than one removal per recorded NIC. The record is written - /// after the interface exists, so a VMM killed in between leaves a TAP - /// nothing on disk points at; a lost or unreadable record reads as an empty - /// list, which used to mean "nothing to remove"; and a manifest that lost a - /// NIC leaves an index the list no longer reaches. netd derives the names - /// instead, so none of that has to be true for teardown to work. + /// Unconditional and non-fatal, which is one decision made twice. The + /// release path must not be gated on a predicate that can change under it: + /// `needs_netd_interface` reads the VM's *current* backend, and a VM whose + /// NIC was a bridge when its TAP was built and is a user-mode NIC now would + /// skip the release for interfaces that exist. And a VM must be able to + /// stop when the daemon holding its interfaces cannot be reached, or a + /// netd outage becomes a fleet that cannot be stopped. /// - /// `networks` now only decides whether to ask at all. An unreachable netd - /// is not a failure: most nodes run none, and stopping a VM must not depend - /// on one being up. - pub(crate) async fn remove_filtered_networks( - &self, - vm_id: &str, - networks: &[Networking], - ) -> Result<()> { - // An empty list is not "no interfaces", it is "no record" -- exactly - // the case a sweep exists for. A record that names only backends netd - // never touches is the one case worth skipping. - let recorded_none = !networks.is_empty() - && networks - .iter() - .all(|network| netd_teardown(network, &self.config.cvm).is_none()); - if recorded_none { - return Ok(()); + /// Nothing is lost by not failing: the next launch releases before it + /// prepares, and startup reconciliation collects what no launch will ever + /// reach. `recorded` is not the source of truth -- it is what a netd too + /// old to sweep by identity has to be told instead. + pub(crate) async fn release_vm_interfaces(&self, vm_id: &str, recorded: &[Networking]) { + let reachability = self.netd_capabilities().await; + if !reachability.is_reachable() { + debug!(vm_id, "no netd to release interfaces from"); + return; + } + if reachability.supports("remove_all") { + match netd::remove_all( + &self.config.netd.socket, + &self.config.cvm.instance_id, + vm_id, + ) + .await + { + Ok(0) => {} + Ok(removed) => info!(vm_id, removed, "released netd-managed interfaces"), + Err(error) if netd::is_unreachable(&error) => { + debug!(vm_id, %error, "no netd to release interfaces from") + } + Err(error) => { + warn!(vm_id, "failed to release netd-managed interfaces: {error:#}") + } + } + return; } - match netd::remove_all( - &self.config.netd.socket, - &self.config.cvm.instance_id, + self.release_recorded_interfaces(vm_id, recorded).await; + } + + /// The teardown a netd that cannot sweep by identity gets. + /// + /// One `Remove` per index, over the record plus a margin for what the + /// record has lost. See [`LEGACY_TEARDOWN_SPAN`]. + async fn release_recorded_interfaces(&self, vm_id: &str, recorded: &[Networking]) { + warn!( vm_id, - ) - .await - { - Ok(0) => Ok(()), - Ok(removed) => { - info!(vm_id, removed, "removed netd-managed interfaces"); - Ok(()) - } - Err(error) if netd::is_unreachable(&error) => { - debug!(vm_id, %error, "no netd to remove interfaces from"); - Ok(()) + "netd on this node cannot release a VM's interfaces by identity; falling back to \ + the recorded ones. Interfaces it has no record of are left behind until this node \ + runs a netd that can sweep" + ); + for nic_index in (0..recorded.len().max(LEGACY_TEARDOWN_SPAN)).rev() { + // Beyond the record there is nothing to say whether the interface + // carried a binding, and an unfiltered removal still clears one it + // finds. Inside it, say what was built. + let filtered = recorded + .get(nic_index) + .and_then(|network| netd_teardown(network, &self.config.cvm)) + .unwrap_or(false); + let identity = InterfaceIdentity { + instance_id: self.config.cvm.instance_id.clone(), + vm_id: vm_id.to_string(), + nic_index, + }; + if let Err(error) = netd::request( + &self.config.netd.socket, + &NetdRequest::Remove { identity, filtered }, + ) + .await + { + if netd::is_unreachable(&error) { + return; + } + warn!(vm_id, nic_index, "failed to release interface: {error:#}"); } - Err(error) => Err(error).context("failed to remove netd-managed networking"), } } @@ -1045,9 +1119,7 @@ impl App { let runtime_networks = self.work_dir(id)?.runtime_networks(); { let _launch = self.launch_lock(id).await; - if let Err(error) = self.remove_filtered_networks(id, &runtime_networks).await { - warn!(id, %error, "failed to remove filtered networking during VM removal"); - } + self.release_vm_interfaces(id, &runtime_networks).await; } // Only delete the workdir for user-initiated removal or if .removing marker exists. @@ -2173,6 +2245,88 @@ mod tests { App::new(config, SupervisorClient::new("http://127.0.0.1:0")) } + fn app_talking_to(netd_socket: &Path) -> App { + use rocket::figment::providers::Format as _; + let mut config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + config.netd.socket = netd_socket.to_path_buf(); + config.cvm.instance_id = "test-instance".to_string(); + App::new(config, SupervisorClient::new("http://127.0.0.1:0")) + } + + /// A netd outage must not become a fleet that cannot be stopped. + #[tokio::test] + async fn a_stop_survives_a_netd_that_is_not_there() { + let app = app_talking_to(Path::new("/nonexistent/dstack-netd.sock")); + // Returns rather than propagating: there is no error type here on + // purpose, because there is no caller that should act on one. + app.release_vm_interfaces("vm-1", &[]).await; + } + + /// The regression this pair exists to prevent: `remove_all` is an + /// operation, and an operation a netd does not have answers with an error + /// that looks exactly like the sweep having failed. Reading that as failure + /// used to fail the stop *and* leave every interface behind -- strictly + /// worse than the per-NIC removal it replaced. + #[tokio::test] + async fn a_netd_too_old_to_sweep_gets_the_teardown_it_understands() { + let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); + let app = app_talking_to(netd.socket()); + app.release_vm_interfaces("vm-1", &[]).await; + + let operations = netd.operations(); + assert_eq!(operations[0], "hello", "capability is asked, not inferred"); + assert!( + !operations.contains(&"remove_all".to_string()), + "an operation it does not have is not sent" + ); + assert_eq!( + operations.iter().filter(|op| *op == "remove").count(), + LEGACY_TEARDOWN_SPAN, + "the record is not the only thing reached, even on a netd that cannot sweep" + ); + } + + #[tokio::test] + async fn a_netd_that_sweeps_is_asked_once_and_by_identity() { + let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ + "hello", + "remove", + "remove_all", + ])); + let app = app_talking_to(netd.socket()); + app.release_vm_interfaces("vm-1", &[]).await; + + assert_eq!(netd.operations(), vec!["hello", "remove_all"]); + let sweep = &netd.seen()[1]; + assert_eq!(sweep["vm_id"], "vm-1"); + assert_eq!(sweep["instance_id"], "test-instance"); + // A sweep names no NIC: reaching the indices the caller can no longer + // name is the entire point. + assert!(sweep.get("nic_index").is_none()); + } + + /// One probe covers a window rather than one call, or netd's serialized + /// accept loop services a connection per VM per operation. + #[tokio::test] + async fn the_capability_answer_is_reused() { + let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ + "hello", + "remove_all", + ])); + let app = app_talking_to(netd.socket()); + app.release_vm_interfaces("vm-1", &[]).await; + app.release_vm_interfaces("vm-2", &[]).await; + assert_eq!( + netd.operations(), + vec!["hello", "remove_all", "remove_all"], + "asked once, acted on twice" + ); + } + /// The window a launch spends between reading "not running" and actually /// starting QEMU is long -- a GPU reset, a netd conversation -- and the /// sweep inside it deletes interfaces by deriving their names. Two entrants diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 180c0102b..98f113c08 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -1020,9 +1020,8 @@ impl VmmRpc for RpcHandler { // must not overlap a launch of the same VM. let _launch = self.app.launch_lock(&request.id).await; self.app - .remove_filtered_networks(&request.id, &runtime_networks) - .await - .context("failed to remove previous filtered networking")?; + .release_vm_interfaces(&request.id, &runtime_networks) + .await; vm_work_dir.clear_runtime_networks()?; } manifest.networks = networks; diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 1415eadae..f4fb28fb2 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -37,6 +37,8 @@ use crate::config::{NetdConfig, NetworkFilterConfig}; const MAX_MESSAGE_SIZE: u64 = 64 * 1024; const CONNECTION_TIMEOUT: Duration = Duration::from_secs(35); +/// How long a capability probe waits. See [`probe`]. +const PROBE_TIMEOUT: Duration = Duration::from_secs(5); const COMMAND_TIMEOUT: Duration = Duration::from_secs(30); const IP_PATH: &str = "/usr/sbin/ip"; const VIRSH_PATH: &str = "/usr/bin/virsh"; @@ -48,6 +50,15 @@ const MAX_QUEUES: u32 = 64; /// whole-VM sweep has to enumerate, since it derives names instead of reading a /// record. const MAX_NIC_INDEX: usize = 255; +/// Every operation this netd accepts, answered to `hello`. +const OPERATIONS: &[&str] = &[ + "hello", + "prepare_bridge", + "prepare_macvtap", + "remove", + "remove_all", + "check", +]; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InterfaceIdentity { @@ -129,6 +140,44 @@ pub struct IngressBinding { pub guest_port: u16, } +/// What a netd can do, asked rather than inferred from a failure. +/// +/// `queues` and `ingress` report a *missing feature* by leaving a response +/// field out, which works because both ride on an operation every netd has. An +/// operation a netd does not have cannot answer that way: it fails, and a +/// failure is indistinguishable from the operation failing for a real reason. +/// A caller left to guess from the message either turns a missing feature into +/// an outage or turns a real failure into silence -- and teardown, where the +/// consequence of guessing wrong is a leaked host interface, is exactly where +/// neither is acceptable. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Capabilities { + /// Implementation and version, for the operator's log. Never parsed: + /// `operations` is what decisions are made from. + #[serde(default)] + pub version: String, + /// Every operation this netd accepts. + #[serde(default)] + pub operations: Vec, + /// Whether it forwards host ports at all. A prepare's `ingress` field still + /// says what one interface actually got; this says whether asking is + /// meaningful, which is what deployment has to know before any prepare + /// exists to read. + #[serde(default)] + pub ingress: bool, + /// Whether it records ownership on the interface itself. A whole-host + /// collection cannot tell one VMM instance's interfaces from another's + /// without it, so a netd that says no is never asked to collect. + #[serde(default)] + pub attribution: bool, +} + +impl Capabilities { + pub fn supports(&self, operation: &str) -> bool { + self.operations.iter().any(|name| name == operation) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrepareMacvtapRequest { #[serde(flatten)] @@ -151,6 +200,13 @@ pub struct PrepareMacvtapRequest { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "operation", rename_all = "snake_case")] pub enum Request { + /// Ask what this netd can do, before asking it to do anything. + /// + /// Cheap by construction: answered without taking the operation lock, so a + /// caller learns what netd can do without first waiting for what it is + /// doing. It doubles as the liveness probe -- a netd that answers is up, + /// and one that predates this answers an error, which is still an answer. + Hello, PrepareBridge(PrepareBridgeRequest), PrepareMacvtap(PrepareMacvtapRequest), Remove { @@ -203,41 +259,70 @@ struct Response { /// How many interfaces a whole-VM sweep deleted. #[serde(default, skip_serializing_if = "Option::is_none")] removed: Option, + /// What this netd can do. Absent from one that predates `hello`, which is + /// the same reading `queues` and `ingress` get. + #[serde(default, skip_serializing_if = "Option::is_none")] + capabilities: Option, #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, } -/// What netd built, echoed back so the caller can verify it matches the -/// request before handing the interface to QEMU. -struct Prepared { - /// The interface this names, absent for an operation that names none. - tap: Option, - device: Option, - queues: Option, - ingress: Option>, - removed: Option, +/// What one request produced. +/// +/// An enum rather than a struct of options, because the shapes do not overlap: +/// a prepare names an interface, a sweep counts them, a probe names neither. +/// One flat [`Response`] still carries all of them on the wire, so a netd that +/// grows an operation stays readable to a caller that does not know it. +enum Outcome { + /// One interface, named because the caller hands that name to QEMU. + Interface { + tap: String, + device: Option, + queues: Option, + ingress: Option>, + }, + /// A sweep names no single interface, so it reports how many it deleted. + Swept { removed: usize }, + Hello(Capabilities), } -impl Prepared { +impl Outcome { fn tap(tap: String) -> Self { - Self { - tap: Some(tap), + Self::Interface { + tap, device: None, queues: None, ingress: None, - removed: None, } } - /// A sweep names no single interface, so it reports how many it deleted. - fn removed(removed: usize) -> Self { - Self { + fn into_response(self) -> Response { + let mut response = Response { + ok: true, tap: None, device: None, queues: None, ingress: None, - removed: Some(removed), + removed: None, + capabilities: None, + error: None, + }; + match self { + Self::Interface { + tap, + device, + queues, + ingress, + } => { + response.tap = Some(tap); + response.device = device; + response.queues = queues; + response.ingress = ingress; + } + Self::Swept { removed } => response.removed = Some(removed), + Self::Hello(capabilities) => response.capabilities = Some(capabilities), } + response } } @@ -282,8 +367,14 @@ impl std::fmt::Display for Unreachable { impl std::error::Error for Unreachable {} /// Whether this error means netd was never reached. +/// +/// `downcast_ref` rather than a walk over `chain()`: a marker attached with +/// `context` is not a link in the source chain, it is the context *of* a link, +/// and `chain()` yields the wrapper rather than the marker inside it. Asking +/// the chain therefore always answered no -- which made every "an unreachable +/// netd is not a failure" branch in this crate unreachable itself. pub fn is_unreachable(error: &anyhow::Error) -> bool { - error.chain().any(|cause| cause.is::()) + error.downcast_ref::().is_some() } pub async fn request(socket: &Path, request: &Request) -> Result { @@ -300,19 +391,103 @@ pub async fn request(socket: &Path, request: &Request) -> Result Result { let request = Request::RemoveAll { instance_id: instance_id.to_string(), vm_id: vm_id.to_string(), }; - Ok(exchange(socket, &request) + exchange(socket, &request) .await? .removed - .unwrap_or_default()) + .context("netd answered a sweep without saying what it removed") +} + +/// This node's netd, as far as it can be asked. +#[derive(Debug, Clone)] +pub enum Reachability { + /// Nothing is listening. Not a failure: most nodes run no netd, and a VM + /// that needs none never notices. + Unreachable, + /// Reached, but it predates `hello`. Everything in [`OPERATIONS`] before + /// `hello` was added is assumed; nothing after it is. + Legacy, + Capable(Capabilities), +} + +impl Reachability { + /// Whether netd answered at all. + pub fn is_reachable(&self) -> bool { + !matches!(self, Self::Unreachable) + } + + pub fn supports(&self, operation: &str) -> bool { + match self { + Self::Unreachable | Self::Legacy => false, + Self::Capable(capabilities) => capabilities.supports(operation), + } + } + + /// Whether asking this netd to forward host ports is meaningful. Unknown + /// counts as no: a caller that assumed yes would report ports as published + /// on the strength of never having asked. + pub fn forwards_ingress(&self) -> bool { + matches!(self, Self::Capable(capabilities) if capabilities.ingress) + } + + pub fn describe(&self) -> String { + match self { + Self::Unreachable => "unreachable".to_string(), + Self::Legacy => "reachable, predates capability reporting".to_string(), + Self::Capable(capabilities) => { + format!("{} [{}]", capabilities.version, capabilities.operations.join(" ")) + } + } + } +} + +/// Asks what this node's netd can do. +/// +/// Never fails: not being there, and being there but too old to say, are both +/// answers a caller has to act on rather than propagate. Both are also +/// distinguishable here and nowhere else -- an error from any other operation +/// cannot tell "netd refused this" from "netd does not have this". +pub async fn probe(socket: &Path) -> Reachability { + // Bounded well below the request timeout. Every teardown asks, and a netd + // that accepts a connection and then stops answering must not turn each of + // them into a thirty-second stall; not answering promptly is, for the + // caller's purposes, the same as not being there. + let answer = match timeout(PROBE_TIMEOUT, exchange(socket, &Request::Hello)).await { + Ok(answer) => answer, + Err(_) => { + warn!("netd accepted a connection but did not answer hello in time"); + return Reachability::Unreachable; + } + }; + match answer { + Ok(response) => match response.capabilities { + Some(capabilities) => Reachability::Capable(capabilities), + // It answered `hello` with nothing to say, which is not a shape + // this netd produces. Treat it as the older protocol rather than + // trusting an empty capability set. + None => Reachability::Legacy, + }, + // It answered, so it is up; it just does not know the question. + Err(error) if !is_unreachable(&error) => { + debug!("netd does not answer hello: {error:#}"); + Reachability::Legacy + } + Err(_) => Reachability::Unreachable, + } } async fn exchange(socket: &Path, request: &Request) -> Result { let operation = match request { + Request::Hello => "hello", Request::PrepareBridge(_) => "prepare_bridge", Request::PrepareMacvtap(_) => "prepare_macvtap", Request::Remove { .. } => "remove", @@ -430,15 +605,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul Err(error) => Err(error), }; let response = match outcome { - Ok(prepared) => Response { - ok: true, - tap: prepared.tap, - device: prepared.device, - queues: prepared.queues, - ingress: prepared.ingress, - removed: prepared.removed, - error: None, - }, + Ok(outcome) => outcome.into_response(), Err(error) => { warn!(%error, "netd request failed"); Response { @@ -448,6 +615,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul queues: None, ingress: None, removed: None, + capabilities: None, error: Some(format!("{error:#}")), } } @@ -476,10 +644,31 @@ async fn read_request(stream: &mut UnixStream) -> Result> { .context("invalid netd request") } -fn handle_request(config: &NetdConfig, request: Request) -> Result { +/// What this build of netd can do. See [`Capabilities`]. +fn capabilities() -> Capabilities { + Capabilities { + version: format!("dstack-netd {}", env!("CARGO_PKG_VERSION")), + operations: OPERATIONS.iter().map(|name| name.to_string()).collect(), + // This netd builds interfaces; it is not the host's forwarder. The + // response field says so per prepare; this says so before one. + ingress: false, + attribution: true, + } +} + +fn handle_request(config: &NetdConfig, request: Request) -> Result { + // Answered before the lock. A caller asks this to find out whether netd + // can do the thing it is about to ask for, and making that wait behind a + // running collection would put a whole-host sweep in front of every + // launch's first question. + if matches!(request, Request::Hello) { + return Ok(Outcome::Hello(capabilities())); + } let libvirt_uri = config.libvirt_uri.as_str(); let _lock = OperationLock::acquire()?; match request { + // Handled above, before the lock. + Request::Hello => Ok(Outcome::Hello(capabilities())), Request::PrepareBridge(request) => { prepare_bridge(libvirt_uri, &request, config.filter_policy()) } @@ -488,13 +677,13 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { } Request::RemoveAll { instance_id, vm_id } => { let removed = sweep_vm_interfaces(libvirt_uri, &instance_id, &vm_id)?; - Ok(Prepared::removed(removed)) + Ok(Outcome::Swept { removed }) } Request::Remove { identity, filtered } => { validate_identity(&identity)?; let tap = tap_name(&identity); remove_interface(libvirt_uri, &tap, binding_cleanup(filtered))?; - Ok(Prepared::tap(tap)) + Ok(Outcome::tap(tap)) } Request::Check { identity, filtered } => { validate_identity(&identity)?; @@ -507,7 +696,7 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { if filtered && !is_macvtap(&tap) { virsh(libvirt_uri, &["nwfilter-binding-dumpxml", &tap], None)?; } - Ok(Prepared::tap(tap)) + Ok(Outcome::tap(tap)) } } } @@ -516,7 +705,7 @@ fn prepare_macvtap( libvirt_uri: &str, request: &PrepareMacvtapRequest, filter: &NetworkFilterConfig, -) -> Result { +) -> Result { let identity = &request.identity; let parent = request.parent.as_str(); let qemu_uid = request.qemu_uid; @@ -586,12 +775,11 @@ fn prepare_macvtap( match result { Ok(device) => { info!(%tap, %parent, %mode, %device, %queues, "prepared macvtap"); - Ok(Prepared { - tap: Some(tap), + Ok(Outcome::Interface { + tap, device: Some(device), queues: Some(queues), ingress: None, - removed: None, }) } Err(error) => { @@ -633,7 +821,7 @@ fn prepare_bridge( libvirt_uri: &str, request: &PrepareBridgeRequest, filter: &NetworkFilterConfig, -) -> Result { +) -> Result { validate_prepare_bridge(request, filter)?; let filtered = request.filtered; let tap = tap_name(&request.identity); @@ -669,15 +857,14 @@ fn prepare_bridge( return Err(error); } info!(%tap, bridge = %request.bridge, %filtered, %queues, "prepared TAP"); - Ok(Prepared { - tap: Some(tap), + Ok(Outcome::Interface { + tap, device: None, queues: Some(queues), // This netd builds interfaces; it is not the host's forwarder. Saying // nothing here is what tells the caller that, so ports it asked for are // reported as unmet rather than assumed done. ingress: None, - removed: None, }) } @@ -1077,6 +1264,174 @@ fn prepare_socket_path(socket: &Path) -> Result<()> { Ok(()) } +/// A netd that exists only to be talked to. +/// +/// The VMM's side of this protocol -- what it falls back to, what it refuses, +/// what it does when the answer is missing -- had no test at all, because +/// every path needed a privileged daemon. It does not: it needs something that +/// answers on a socket. This is that, scripted per behaviour, recording what +/// it was asked so a test can assert on the conversation rather than on its +/// effects. +#[cfg(test)] +pub(crate) mod testing { + use std::{ + path::{Path, PathBuf}, + sync::{Arc, Mutex}, + }; + + use serde_json::{json, Value}; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::UnixListener, + }; + + /// How the fake answers. + #[derive(Debug, Clone)] + pub(crate) enum Behavior { + /// Answers `hello` with the given operations, and every listed + /// operation with a plausible success. + Capable { + operations: Vec, + ingress: bool, + }, + /// Reached, but predates `hello`: every unknown operation is an error, + /// exactly as `serde` produces one. + Legacy, + } + + impl Behavior { + pub(crate) fn capable(operations: &[&str]) -> Self { + Self::Capable { + operations: operations.iter().map(|name| name.to_string()).collect(), + ingress: false, + } + } + + pub(crate) fn forwarding(operations: &[&str]) -> Self { + match Self::capable(operations) { + Self::Capable { operations, .. } => Self::Capable { + operations, + ingress: true, + }, + other => other, + } + } + } + + pub(crate) struct FakeNetd { + _dir: tempfile::TempDir, + socket: PathBuf, + seen: Arc>>, + } + + impl FakeNetd { + pub(crate) fn spawn(behavior: Behavior) -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("netd.sock"); + let listener = UnixListener::bind(&socket).expect("bind"); + let seen = Arc::new(Mutex::new(Vec::new())); + let recorder = seen.clone(); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let behavior = behavior.clone(); + let recorder = recorder.clone(); + tokio::spawn(async move { + let mut message = Vec::new(); + if stream.read_to_end(&mut message).await.is_err() { + return; + } + let Ok(request) = serde_json::from_slice::(&message) else { + return; + }; + recorder.lock().expect("poisoned").push(request.clone()); + let response = answer(&behavior, &request); + let _ = stream.write_all(&serde_json::to_vec(&response).unwrap()).await; + let _ = stream.shutdown().await; + }); + } + }); + Self { + _dir: dir, + socket, + seen, + } + } + + pub(crate) fn socket(&self) -> &Path { + &self.socket + } + + /// Every request it was sent, in order. + pub(crate) fn seen(&self) -> Vec { + self.seen.lock().expect("poisoned").clone() + } + + pub(crate) fn operations(&self) -> Vec { + self.seen() + .iter() + .map(|request| request["operation"].as_str().unwrap_or("?").to_string()) + .collect() + } + } + + fn answer(behavior: &Behavior, request: &Value) -> Value { + let operation = request["operation"].as_str().unwrap_or_default(); + let (operations, ingress) = match behavior { + Behavior::Legacy => { + return match operation { + // What the real thing answers for an operation it knows. + "prepare_bridge" | "prepare_macvtap" | "remove" | "check" => { + json!({"ok": true, "tap": "dtdeadbeef00"}) + } + other => json!({ + "ok": false, + "error": format!("invalid netd request: unknown variant `{other}`"), + }), + }; + } + Behavior::Capable { + operations, + ingress, + } => (operations, *ingress), + }; + if !operations.iter().any(|name| name == operation) { + return json!({ + "ok": false, + "error": format!("invalid netd request: unknown variant `{operation}`"), + }); + } + match operation { + "hello" => json!({ + "ok": true, + "capabilities": { + "version": "fake-netd", + "operations": operations, + "ingress": ingress, + "attribution": true, + }, + }), + "prepare_bridge" | "prepare_macvtap" => { + let mut response = json!({ + "ok": true, + "tap": "dtdeadbeef00", + "queues": request["queues"].as_u64().unwrap_or(1).max(1), + }); + if ingress { + let asked = request["ingress"].as_array().cloned().unwrap_or_default(); + response["ingress"] = Value::Array(asked); + } + response + } + "remove" | "check" => json!({"ok": true, "tap": "dtdeadbeef00"}), + "remove_all" => json!({"ok": true, "removed": 0}), + _ => json!({"ok": true}), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -1586,30 +1941,94 @@ mod tests { /// means nothing. #[test] fn a_sweep_reports_a_count_and_no_interface() { - let response = Response { - ok: true, - tap: Prepared::removed(3).tap, - device: None, - queues: None, - ingress: None, - removed: Some(3), - error: None, - }; - let value = serde_json::to_value(&response).unwrap(); + let value = serde_json::to_value(Outcome::Swept { removed: 3 }.into_response()).unwrap(); assert_eq!(value["removed"], 3); assert!(value.get("tap").is_none()); // A prepare still names one, because the caller hands that name to QEMU. - let value = serde_json::to_value(Response { - ok: true, - tap: Prepared::tap("dtabc".into()).tap, - device: None, - queues: None, - ingress: None, - removed: None, - error: None, - }) - .unwrap(); + let value = + serde_json::to_value(Outcome::tap("dtabc".into()).into_response()).unwrap(); assert_eq!(value["tap"], "dtabc"); + assert!(value.get("removed").is_none()); + } + + /// A netd that answers a sweep with no count did not sweep. Reading the + /// absent field as zero is the same conflation `queues` and `ingress` are + /// shaped to avoid, and here it would report a netd that cannot collect a + /// VM's interfaces as a VM that had none. + #[tokio::test] + async fn a_sweep_without_a_count_is_not_read_as_an_empty_one() { + // Claims the operation, answers without the field. + let netd = testing::FakeNetd::spawn(testing::Behavior::capable(&["hello"])); + let error = remove_all(netd.socket(), "instance", "vm") + .await + .expect_err("an answer with no count is not a successful sweep"); + assert!(!is_unreachable(&error)); + + let netd = testing::FakeNetd::spawn(testing::Behavior::capable(&["hello", "remove_all"])); + assert_eq!(remove_all(netd.socket(), "instance", "vm").await.unwrap(), 0); + } + + /// Every "an unreachable netd is not a failure" branch in the VMM hangs + /// off this one predicate, and a marker attached as context is not a link + /// in the source chain. + #[test] + fn an_unreachable_netd_is_recognized_through_the_contexts_stacked_on_it() { + let error = anyhow::Error::from(std::io::Error::from(std::io::ErrorKind::NotFound)) + .context(Unreachable) + .context("failed to connect to netd at /run/netd.sock") + .context("failed to prepare netd-managed networking"); + assert!(is_unreachable(&error)); + + let other = anyhow::anyhow!("netd remove_all failed: no such bridge") + .context("failed to prepare netd-managed networking"); + assert!(!is_unreachable(&other)); + } + + #[tokio::test] + async fn a_probe_tells_absent_from_old_from_answering() { + assert!(matches!( + probe(Path::new("/nonexistent/netd.sock")).await, + Reachability::Unreachable + )); + + let legacy = testing::FakeNetd::spawn(testing::Behavior::Legacy); + let reachability = probe(legacy.socket()).await; + assert!(matches!(reachability, Reachability::Legacy)); + // Reached, so a caller must not treat it as absent -- but it can do + // nothing this netd was not already able to do. + assert!(reachability.is_reachable()); + assert!(!reachability.supports("remove_all")); + assert!(!reachability.forwards_ingress()); + + let netd = testing::FakeNetd::spawn(testing::Behavior::forwarding(&["hello", "remove_all"])); + let reachability = probe(netd.socket()).await; + assert!(reachability.supports("remove_all")); + assert!(!reachability.supports("gc")); + assert!(reachability.forwards_ingress()); + } + + /// The whole point of asking: a caller must be able to tell an operation + /// this netd does not have from one that failed, and the two look the same + /// in an error message. + #[test] + fn a_probe_reports_what_this_netd_can_do() { + let value = serde_json::to_value(&Request::Hello).unwrap(); + assert_eq!(value["operation"], "hello"); + + let response = Outcome::Hello(capabilities()).into_response(); + let capabilities = response.capabilities.expect("hello answers capabilities"); + assert!(capabilities.supports("remove_all")); + assert!(capabilities.supports("hello")); + assert!(!capabilities.supports("forward_the_whole_internet")); + // This netd builds interfaces and does not forward host ports. Saying + // so before a prepare is what lets deployment refuse a mapping it + // cannot honour, instead of a launch warning about it afterwards. + assert!(!capabilities.ingress); + assert!(capabilities.attribution); + + // Absent capabilities is an older netd, not one that can do nothing. + let legacy: Response = serde_json::from_str(r#"{"ok":true}"#).unwrap(); + assert!(legacy.capabilities.is_none()); } } From d49359d5ea9e85b6286102820db3bb3fcfdcb011 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 02:11:13 -0700 Subject: [PATCH 05/34] feat(netd): record who an interface belongs to, on the interface Deriving a name answers "where is this VM's interface". It cannot answer "whose is this interface", and that is the question a leak is made of: a VM whose directory was deleted, a VMM instance that was decommissioned, an interface built before an upgrade. A digest is not reversible, so nothing on the host could attribute one -- and with several VMM instances sharing a netd, nothing could even tell whose it was to collect. So write it down where it cannot drift: `ip link set dev alias`. The kernel holds it for exactly the interface's lifetime. A file under /run would be a second thing to keep in step with the first, and a record that got out of step -- written after the interface, lost with the directory -- is the failure being fixed, not the fix. The record is a hint, never an authority. Anything that can reach this socket can assert an identity, and the interface name is a digest of one, so a record is believed only when re-deriving the name from it reproduces the name it is written on. Forged, truncated, ambiguous and absent all fail that one check and land in the same bucket, which is the bucket a collection has to treat conservatively anyway. `list` then enumerates: the reserved `dt` name space, narrowed by the kernel's own answer about what kind of device it is, so netd will not offer up something it did not create. Orphaned nwfilter bindings are listed too -- a binding outlives the interface it was bound to, so an inventory that only looked at interfaces would miss the one piece of state that survives them. Absent rather than empty on a netd that cannot enumerate: "I hold nothing" and "I cannot say" are not the same answer. `dstack-vmm netd list` reads it. An operator could not previously see what netd held at all. One new refusal: an identity whose record would exceed the kernel's 255-byte alias, or whose instance ID contains the separator. Both would produce an interface nothing can attribute, which is the state this exists to stop creating. Real identities are a `path-` digest and a VM ID: about sixty bytes. --- dstack/vmm/src/main.rs | 63 +++++++++ dstack/vmm/src/netd.rs | 302 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 364 insertions(+), 1 deletion(-) diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 17b93d194..f645a94e8 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -77,6 +77,23 @@ struct NetdArgs { /// Override the Unix socket configured in [netd]. #[arg(long)] socket: Option, + /// Inspect a running netd instead of starting one. + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum NetdCommand { + /// List every host interface netd holds. + /// + /// Answers the question a leak is made of -- whose is this interface -- + /// which deriving a name from an identity cannot. + List { + /// Only this VMM instance's interfaces. Defaults to every one netd + /// owns, including those it cannot attribute. + #[arg(long)] + instance: Option, + }, } #[derive(ClapArgs)] @@ -192,6 +209,49 @@ async fn log_rotation_task(app: App) { } } +/// Client-side netd subcommands. Talks to the socket like the VMM does, so it +/// needs whatever the socket's permissions ask for and not root. +async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result<()> { + match command { + NetdCommand::List { instance } => { + let interfaces = netd::list(&config.socket, instance.as_deref().unwrap_or_default()) + .await + .context("failed to list netd interfaces")?; + println!( + "{:<16} {:<8} {:<24} {:<38} {:>3} {}", + "INTERFACE", "KIND", "INSTANCE", "VM", "NIC", "FILTERED" + ); + let mut unattributed = 0; + for record in &interfaces { + if record.instance_id.is_none() { + unattributed += 1; + } + println!( + "{:<16} {:<8} {:<24} {:<38} {:>3} {}", + record.tap, + record.kind, + record.instance_id.as_deref().unwrap_or("-"), + record.vm_id.as_deref().unwrap_or("-"), + record + .nic_index + .map_or_else(|| "-".to_string(), |index| index.to_string()), + if record.bound { "yes" } else { "no" }, + ); + } + println!("\n{} interface(s)", interfaces.len()); + if unattributed > 0 { + // Not a fault to fix by hand: an interface built before netd + // recorded ownership, or by another netd, carries no record and + // gets one the next time its VM launches. + println!( + "{unattributed} carry no ownership record, so a collection will not touch them" + ); + } + Ok(()) + } + } +} + #[rocket::main] async fn main() -> Result<()> { { @@ -235,6 +295,9 @@ async fn main() -> Result<()> { .context("failed to load [cvm.network_filter] for netd")?, ); } + if let Some(command) = &netd_args.command { + return run_netd_command(&netd_config, command).await; + } return netd::serve(netd_config).await; } diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index f4fb28fb2..b5a7e004d 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -57,8 +57,19 @@ const OPERATIONS: &[&str] = &[ "prepare_macvtap", "remove", "remove_all", + "list", "check", ]; +/// The interface names netd may create. Reserved: anything matching it is +/// netd's to delete, and nothing else on the host may take one. +const TAP_PREFIX: &str = "dt"; +/// Hex characters of digest in a TAP name, after [`TAP_PREFIX`]. +const TAP_DIGEST_CHARS: usize = 12; +/// Version tag on the ownership record. Present so a later format can be told +/// from this one rather than mis-parsed as it. +const ALIAS_PREFIX: &str = "dstack1"; +/// What the kernel stores in an interface alias, minus the terminator. +const MAX_IFALIAS: usize = 255; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InterfaceIdentity { @@ -140,6 +151,39 @@ pub struct IngressBinding { pub guest_port: u16, } +/// One host resource netd holds. +/// +/// `instance_id` and `vm_id` are absent when the interface carries no record +/// that checks out: built by a netd too old to write one, by a third-party +/// netd, or by this one in the instant between creating the interface and +/// recording it. Absent is not "nobody's" -- it is "not known to be anybody's", +/// which is a materially different thing to a collection. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InterfaceRecord { + pub tap: String, + /// `"tap"`, `"macvtap"`, or `"binding"` for an nwfilter binding whose + /// interface is already gone. A binding outlives the interface it was + /// bound to, so a collection that only looked at interfaces would leave + /// the one piece of state that survives them. + pub kind: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub instance_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vm_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nic_index: Option, + /// Whether libvirt holds an nwfilter binding at this name. + #[serde(default)] + pub bound: bool, +} + +impl InterfaceRecord { + /// Whether this is recorded as belonging to one VM of one VMM instance. + pub fn belongs_to(&self, instance_id: &str, vm_id: &str) -> bool { + self.instance_id.as_deref() == Some(instance_id) && self.vm_id.as_deref() == Some(vm_id) + } +} + /// What a netd can do, asked rather than inferred from a failure. /// /// `queues` and `ingress` report a *missing feature* by leaving a response @@ -229,6 +273,20 @@ pub enum Request { instance_id: String, vm_id: String, }, + /// Everything netd holds, so that an operator -- and a reconciliation -- + /// can see the host's interfaces without being told what to look for. + /// + /// Deriving a name answers "where is this VM's interface". It cannot + /// answer "whose is this interface", which is the question a leak is made + /// of: a VM whose directory was deleted, a VMM instance that was + /// decommissioned, an interface built by a netd that has since been + /// upgraded. Enumeration answers it. + List { + /// Only interfaces recorded as this instance's. Empty lists every one + /// netd owns, whatever it is recorded as and whether or not it is. + #[serde(default)] + instance_id: String, + }, /// Verify a deterministic TAP and binding for operations and integration /// diagnostics. The VMM startup path uses Prepare rather than Check. Check { @@ -263,6 +321,11 @@ struct Response { /// the same reading `queues` and `ingress` get. #[serde(default, skip_serializing_if = "Option::is_none")] capabilities: Option, + /// Everything netd holds. Absent, rather than empty, from a netd that + /// cannot enumerate: "I hold nothing" and "I cannot say" are answers a + /// collection must not confuse. + #[serde(default, skip_serializing_if = "Option::is_none")] + interfaces: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, } @@ -284,6 +347,7 @@ enum Outcome { /// A sweep names no single interface, so it reports how many it deleted. Swept { removed: usize }, Hello(Capabilities), + Listed(Vec), } impl Outcome { @@ -305,6 +369,7 @@ impl Outcome { ingress: None, removed: None, capabilities: None, + interfaces: None, error: None, }; match self { @@ -321,6 +386,7 @@ impl Outcome { } Self::Swept { removed } => response.removed = Some(removed), Self::Hello(capabilities) => response.capabilities = Some(capabilities), + Self::Listed(interfaces) => response.interfaces = Some(interfaces), } response } @@ -332,7 +398,58 @@ pub fn tap_name(identity: &InterfaceIdentity) -> String { identity.instance_id, identity.vm_id, identity.nic_index ); let digest = Sha256::digest(input.as_bytes()); - format!("dt{}", hex::encode(&digest[..6])) + format!("{TAP_PREFIX}{}", hex::encode(&digest[..TAP_DIGEST_CHARS / 2])) +} + +/// The ownership record netd writes onto every interface it creates. +/// +/// The record lives on the resource, so it has exactly the resource's +/// lifetime. A file under `/run` would be a second thing to keep in step with +/// the first, and the failure this whole path exists to fix is precisely a +/// record that got out of step: written after the interface, lost with the +/// directory, and unreadable to anything but the process that wrote it. +/// +/// Never trusted as *authority*. Anything that can reach this socket can also +/// name an identity, and the interface name is a digest of that identity -- +/// so a record is believed only when re-deriving the name from it reproduces +/// the name it is written on. Ambiguity (a separator inside an identity), +/// truncation, and forgery all fail that check and land in the same bucket as +/// no record at all, which is the bucket handled conservatively. +pub fn interface_alias(identity: &InterfaceIdentity) -> String { + format!( + "{ALIAS_PREFIX}:{}:{}:{}", + identity.nic_index, identity.instance_id, identity.vm_id + ) +} + +/// The identity an interface claims, if the claim checks out. +/// +/// `nic_index` first, so the two free-form fields are the last two and a +/// `vm_id` containing the separator still parses. An `instance_id` containing +/// one does not, and is refused at prepare rather than mis-parsed here. +pub fn owner_of(tap: &str, alias: &str) -> Option { + let rest = alias.trim().strip_prefix(ALIAS_PREFIX)?.strip_prefix(':')?; + let (nic_index, rest) = rest.split_once(':')?; + let (instance_id, vm_id) = rest.split_once(':')?; + let identity = InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + nic_index: nic_index.parse().ok()?, + }; + // The name is the proof. A record that does not reproduce it describes + // some other interface, or nothing. + (tap_name(&identity) == tap).then_some(identity) +} + +/// Whether this name is one netd can have created. See [`TAP_PREFIX`]. +pub fn is_managed_name(interface: &str) -> bool { + let Some(digest) = interface.strip_prefix(TAP_PREFIX) else { + return false; + }; + digest.len() == TAP_DIGEST_CHARS + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } pub fn instance_id(configured: &str, run_path: &Path) -> String { @@ -407,6 +524,18 @@ pub async fn remove_all(socket: &Path, instance_id: &str, vm_id: &str) -> Result .context("netd answered a sweep without saying what it removed") } +/// Everything netd holds, optionally narrowed to one VMM instance. See +/// [`Request::List`]. +pub async fn list(socket: &Path, instance_id: &str) -> Result> { + let request = Request::List { + instance_id: instance_id.to_string(), + }; + exchange(socket, &request) + .await? + .interfaces + .context("netd answered a listing without one") +} + /// This node's netd, as far as it can be asked. #[derive(Debug, Clone)] pub enum Reachability { @@ -492,6 +621,7 @@ async fn exchange(socket: &Path, request: &Request) -> Result { Request::PrepareMacvtap(_) => "prepare_macvtap", Request::Remove { .. } => "remove", Request::RemoveAll { .. } => "remove_all", + Request::List { .. } => "list", Request::Check { .. } => "check", }; let exchange = async { @@ -616,6 +746,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul ingress: None, removed: None, capabilities: None, + interfaces: None, error: Some(format!("{error:#}")), } } @@ -675,6 +806,10 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { Request::PrepareMacvtap(request) => { prepare_macvtap(libvirt_uri, &request, config.filter_policy()) } + Request::List { instance_id } => Ok(Outcome::Listed(list_interfaces( + libvirt_uri, + &instance_id, + ))), Request::RemoveAll { instance_id, vm_id } => { let removed = sweep_vm_interfaces(libvirt_uri, &instance_id, &vm_id)?; Ok(Outcome::Swept { removed }) @@ -751,6 +886,7 @@ fn prepare_macvtap( add.extend_from_slice(&["type", "macvtap", "mode", mode]); ip(&add)?; let result = (|| { + set_alias(&tap, identity)?; let ifindex = std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifindex")) .context("failed to read macvtap ifindex")?; @@ -840,6 +976,10 @@ fn prepare_bridge( add.extend_from_slice(&["user", &uid]); ip(&add)?; let result = (|| { + // Before anything else it could fail at. An interface that exists + // without a record is one nothing can attribute, and the window in + // which that is true is the window a crash turns permanent. + set_alias(&tap, &request.identity)?; ip(&["link", "set", "dev", &tap, "master", &request.bridge])?; if filtered { let xml = binding_xml(request, &tap, filter); @@ -946,6 +1086,73 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res } } +/// Every resource netd owns, read off the host rather than out of a record. +/// +/// Ownership is the reserved name plus the kernel's own answer about what kind +/// of device it is; attribution is the interface's alias, checked by +/// re-deriving the name from it. A listing never fails for want of libvirt: on +/// a node that does not filter, `libvirtd` need not be running, and an +/// interface inventory that refused to be produced without it would be +/// unavailable exactly where unfiltered TAPs live. +fn list_interfaces(libvirt_uri: &str, instance_id: &str) -> Vec { + let bindings = existing_bindings(libvirt_uri); + let mut records = Vec::new(); + let mut seen = HashSet::new(); + if let Ok(entries) = std::fs::read_dir("/sys/class/net") { + for entry in entries.flatten() { + let Ok(tap) = entry.file_name().into_string() else { + continue; + }; + if !is_managed_name(&tap) { + continue; + } + let kind = if is_macvtap(&tap) { + "macvtap" + } else if is_tuntap(&tap) { + "tap" + } else { + // The name is netd's to use, but this is not a device netd + // creates. Listing it would invite a collection to delete it. + continue; + }; + let alias = std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifalias")) + .unwrap_or_default(); + let owner = owner_of(&tap, &alias); + seen.insert(tap.clone()); + records.push(InterfaceRecord { + bound: bindings + .as_ref() + .is_some_and(|bindings| bindings.contains(&tap)), + kind: kind.to_string(), + nic_index: owner.as_ref().map(|identity| identity.nic_index), + instance_id: owner.as_ref().map(|identity| identity.instance_id.clone()), + vm_id: owner.map(|identity| identity.vm_id), + tap, + }); + } + } + // A binding outlives its interface, and an interface is the only thing + // that carries a record, so an orphaned binding can never be attributed. + // It is still netd's: nothing else creates a binding at one of these names. + for name in bindings.into_iter().flatten() { + if is_managed_name(&name) && !seen.contains(&name) { + records.push(InterfaceRecord { + tap: name, + kind: "binding".to_string(), + instance_id: None, + vm_id: None, + nic_index: None, + bound: true, + }); + } + } + if !instance_id.is_empty() { + records.retain(|record| record.instance_id.as_deref() == Some(instance_id)); + } + records.sort_by(|left, right| left.tap.cmp(&right.tap)); + records +} + fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Result<()> { let macvtap = is_macvtap(tap); if Path::new("/sys/class/net").join(tap).exists() { @@ -985,6 +1192,14 @@ fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Re /// Whether this is a tun/tap device. `tun_flags` is published by the tun /// driver and by nothing else, so its presence is the kernel's own answer -- /// as `macvtap/` is for the other kind of device netd creates. +/// Records who an interface belongs to, on the interface. See +/// [`interface_alias`]. +fn set_alias(tap: &str, identity: &InterfaceIdentity) -> Result<()> { + let alias = interface_alias(identity); + ip(&["link", "set", "dev", tap, "alias", &alias]) + .with_context(|| format!("failed to record ownership on {tap}")) +} + fn is_tuntap(interface: &str) -> bool { Path::new("/sys/class/net") .join(interface) @@ -1104,6 +1319,22 @@ fn validate_identity(identity: &InterfaceIdentity) -> Result<()> { if identity.nic_index > MAX_NIC_INDEX { bail!("NIC index is out of range"); } + // An identity that cannot be recorded on the interface is refused rather + // than built unattributed. A host resource nothing can name the owner of + // is the thing this whole path exists to stop producing, and the kernel's + // alias is the only place with the interface's exact lifetime to put it. + let alias = interface_alias(identity); + if alias.len() > MAX_IFALIAS { + bail!( + "identity is too long to record on the interface: {} bytes of {MAX_IFALIAS}", + alias.len() + ); + } + // The record puts the two free-form fields last, so only the first of them + // has to be unambiguous. + if identity.instance_id.contains(':') { + bail!("instance ID must not contain ':'"); + } Ok(()) } @@ -1952,6 +2183,75 @@ mod tests { assert!(value.get("removed").is_none()); } + /// The record is a hint; the name is the proof. Everything that can go + /// wrong with reading a string off an interface -- forged, truncated, + /// ambiguous, absent -- has to land in the same bucket, and it has to be + /// the bucket a collection treats conservatively. + #[test] + fn an_interface_says_whose_it_is_and_the_name_is_what_proves_it() { + let nic = identity("path-abc", "vm-1", 3); + let tap = tap_name(&nic); + let alias = interface_alias(&nic); + assert_eq!(alias, "dstack1:3:path-abc:vm-1"); + + let owner = owner_of(&tap, &alias).expect("its own record checks out"); + assert_eq!(owner.instance_id, "path-abc"); + assert_eq!(owner.vm_id, "vm-1"); + assert_eq!(owner.nic_index, 3); + + // A record naming some other interface proves nothing about this one. + // This is what makes the record unforgeable without making it + // authoritative: anything that can reach the socket can write a + // string, but only the true identity re-derives the name. + let forged = interface_alias(&identity("path-abc", "someone-elses-vm", 3)); + assert!(owner_of(&tap, &forged).is_none()); + assert!(owner_of(&tap, "").is_none()); + assert!(owner_of(&tap, "dstack1:3:path-abc").is_none()); + assert!(owner_of(&tap, &alias[..alias.len() - 2]).is_none()); + // A format this build does not know is not this format. + assert!(owner_of(&tap, &alias.replace("dstack1", "dstack2")).is_none()); + + // The two free-form fields are last and only the first of them has to + // be unambiguous, so a VM ID carrying the separator still reads back. + let odd = identity("path-abc", "vm:with:colons", 0); + assert_eq!( + owner_of(&tap_name(&odd), &interface_alias(&odd)).map(|owner| owner.vm_id), + Some("vm:with:colons".to_string()) + ); + // An instance ID carrying it is refused instead of mis-parsed. + assert!(validate_identity(&identity("path:abc", "vm-1", 0)).is_err()); + } + + /// An identity that cannot be recorded would produce an interface nothing + /// can attribute, which is the state this whole path exists to stop + /// creating. Refusing it is the only answer that keeps the invariant. + #[test] + fn an_identity_too_long_to_record_is_refused() { + let long = "v".repeat(128); + assert!(validate_identity(&identity("instance", &long, 0)).is_ok()); + let identity_too_long = identity(&"i".repeat(128), &long, 255); + assert!(interface_alias(&identity_too_long).len() > MAX_IFALIAS); + let error = validate_identity(&identity_too_long).unwrap_err().to_string(); + assert!(error.contains("too long to record"), "{error}"); + } + + /// The name space netd claims. A collection deletes what matches, so what + /// matches has to be exactly what netd can produce. + #[test] + fn the_managed_name_space_is_exactly_what_netd_produces() { + assert!(is_managed_name(&tap_name(&identity("instance", "vm", 0)))); + assert!(is_managed_name("dt0123456789ab")); + assert!(!is_managed_name("dt0123456789AB"), "digests are lower case"); + assert!(!is_managed_name("dt0123456789a"), "one short"); + assert!(!is_managed_name("dt0123456789abc"), "one long"); + assert!(!is_managed_name("dtzzzzzzzzzzzz")); + assert!(!is_managed_name("virbr0")); + assert!(!is_managed_name("eth0")); + // The whole space fits in IFNAMSIZ, or the kernel would refuse the + // names this reserves. + assert!(tap_name(&identity("instance", "vm", 255)).len() < 16); + } + /// A netd that answers a sweep with no count did not sweep. Reading the /// absent field as zero is the same conflation `queues` and `ingress` are /// shaped to avoid, and here it would report a netd that cannot collect a From 3be1eb3229fe4ba0606880eea027eed3fe8d54ab Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 02:18:25 -0700 Subject: [PATCH 06/34] feat(vmm): collect host interfaces no VM claims Per-VM teardown reaches only what its caller can still name, and a leak is made of exactly the interfaces nothing names any more: a VM removed while the VMM was down, a workdir deleted by hand, a teardown that raced a netd outage and whose VM no longer exists to retry it. No amount of care at the per-VM call sites reaches those, because the call site is gone. `gc` compares what netd holds against the VMs a VMM instance has. The live set is every VM it has, running or not: a VMM restarts under VMs that keep running, and collecting by what is running would delete their networking out from under them. It runs at startup after the VMs are loaded and before the API is served -- the set is a snapshot, and a VM created between taking it and acting on it would be in netd's listing and not in the set -- and then on an interval, for what accumulates while the VMM is up. Three rules, and the second is why interfaces carry a record at all: - recorded as another instance's: never touched. Two VMM instances share one netd, and without the record a collection cannot tell that instance's running VM from garbage. - recorded as ours, for a VM we no longer have: collected. - no record that checks out: kept by default, and reported. It is not nobody's -- it is an interface from before ownership was recorded, or from another netd. `collect_unattributed` opts in where the operator knows nothing else creates interfaces in netd's name space. The upgrade is safe by construction: a collection derives the names its own live VMs would occupy and keeps those even when told to collect unattributed, so a fleet running from before this existed survives the first pass, and each interface gains a record the next time its VM launches. Both passes are bounded. Under the operation lock and inside a serialized accept loop, an unbounded pass is not slow, it is an outage -- one hung `virsh` per interface holds every other VM's prepare behind it while the caller that asked has long since timed out. A deadline stops the pass and reports what it did; libvirt is asked once per pass rather than once per interface, because asking again after it has failed is how a bounded pass becomes an unbounded one. `dstack-vmm netd remove-vm` for a VM whose VMM will never ask again. --- dstack/vmm/src/app.rs | 116 +++++++++++ dstack/vmm/src/config.rs | 25 +++ dstack/vmm/src/main.rs | 55 ++++- dstack/vmm/src/netd.rs | 433 +++++++++++++++++++++++++++++++++++++-- dstack/vmm/vmm.toml | 11 + 5 files changed, 618 insertions(+), 22 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 819da4fe3..9553750ed 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -930,6 +930,78 @@ impl App { reachability } + /// Deletes every host interface netd holds for a VM this instance no + /// longer has. + /// + /// Per-VM release reaches only what its caller can still name. This reaches + /// what nothing names any more, which is where a leak actually ends up: a + /// VM removed while the VMM was down, a workdir deleted by hand, a + /// teardown that raced a netd outage and was never retried because the VM + /// it belonged to no longer exists to retry it. + /// + /// The live set is every VM this instance has, running or not. Not the + /// running set: a VMM restarts under VMs that keep running, and collecting + /// by what is running would delete their interfaces out from under them. + /// + /// At startup this must run before the API is served -- the set is a + /// snapshot, and a VM created after it was taken would be in netd's + /// listing and not in the snapshot. netd's own lock keeps the collection + /// from overlapping a prepare; the ordering here is what keeps it from + /// racing the decision. + pub(crate) async fn reconcile_netd_interfaces(&self) { + let reachability = self.netd_capabilities().await; + if !reachability.is_reachable() { + debug!("no netd to reconcile interfaces with"); + return; + } + if !reachability.supports("gc") { + warn!( + netd = %reachability.describe(), + "netd on this node cannot collect interfaces no VM claims; a VM removed while \ + this VMM was down leaves its host interfaces behind until it runs a newer netd" + ); + return; + } + let live: Vec = self.lock().vms.keys().cloned().collect(); + let policy = if self.config.netd.collect_unattributed { + netd::UnattributedPolicy::Remove + } else { + netd::UnattributedPolicy::Keep + }; + match netd::collect( + &self.config.netd.socket, + &self.config.cvm.instance_id, + live, + policy, + false, + ) + .await + { + Ok(collection) => { + if collection.removed > 0 { + for record in &collection.collected { + info!( + tap = %record.tap, + vm_id = record.vm_id.as_deref().unwrap_or("-"), + "collected a host interface no VM of this instance claims" + ); + } + info!( + removed = collection.removed, + "reconciled netd-managed interfaces" + ); + } + if collection.incomplete { + warn!("netd stopped collecting on its deadline; the next pass continues"); + } + } + Err(error) if netd::is_unreachable(&error) => { + debug!("no netd to reconcile interfaces with") + } + Err(error) => warn!("failed to reconcile netd-managed interfaces: {error:#}"), + } + } + /// Releases every host interface netd holds for this VM. /// /// Unconditional and non-fatal, which is one decision made twice. The @@ -2327,6 +2399,50 @@ mod tests { ); } + /// A collection is decided against the set of VMs this instance has, and + /// asking for one from a netd that cannot do it must not look like asking + /// for one that found nothing. + #[tokio::test] + async fn reconciliation_is_asked_for_only_where_it_exists() { + let old = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ + "hello", + "remove_all", + ])); + let app = app_talking_to(old.socket()); + app.reconcile_netd_interfaces().await; + assert_eq!( + old.operations(), + vec!["hello"], + "an operation it does not have is not sent" + ); + + let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ + "hello", "gc", + ])); + let app = app_talking_to(netd.socket()); + app.reconcile_netd_interfaces().await; + let request = &netd.seen()[1]; + assert_eq!(request["operation"], "gc"); + assert_eq!(request["instance_id"], "test-instance"); + assert_eq!(request["live_vm_ids"].as_array().unwrap().len(), 0); + assert_eq!(request["dry_run"], false); + // The conservative default: an interface with no ownership record may + // be another VMM instance's running VM. + assert_eq!(request["unattributed"], "keep"); + } + + #[tokio::test] + async fn collecting_unattributed_interfaces_is_the_operators_to_ask_for() { + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&["hello", "gc"])); + let mut app = app_talking_to(netd.socket()); + let mut config = (*app.config).clone(); + config.netd.collect_unattributed = true; + app = App::new(config, SupervisorClient::new("http://127.0.0.1:0")); + app.reconcile_netd_interfaces().await; + assert_eq!(netd.seen()[1]["unattributed"], "remove"); + } + /// The window a launch spends between reading "not running" and actually /// starting QEMU is long -- a GPU reset, a netd conversation -- and the /// sweep inside it deletes interfaces by deriving their names. Two entrants diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index c7542ca84..f6fe7962b 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -644,6 +644,29 @@ pub struct NetdConfig { /// left inferring policy from a file that does not state it. #[serde(default)] pub network_filter: Option, + /// How often the VMM asks netd to collect interfaces no VM of its claims. + /// + /// The pass at startup is not optional and does not read this: it is the + /// one moment the VMM knows every VM it has, and everything a crash left + /// behind is still there. This is the backstop for what accumulates while + /// it runs -- a removal that raced a netd outage, an interface a stop could + /// not reach. Zero turns it off. + #[serde(default = "default_reconcile_interval")] + pub reconcile_interval_secs: u64, + /// Whether a collection may delete interfaces it cannot attribute. + /// + /// Off. An interface carrying no ownership record is not nobody's: on a + /// host where two VMM instances share one netd, it may be the other one's + /// running VM, and before netd recorded ownership every interface looked + /// like this. Turn it on only where nothing else creates interfaces in + /// netd's name space -- a single VMM instance on the host. + #[serde(default)] + pub collect_unattributed: bool, +} + +/// See [`NetdConfig::reconcile_interval_secs`]. +fn default_reconcile_interval() -> u64 { + 3600 } impl Default for NetdConfig { @@ -653,6 +676,8 @@ impl Default for NetdConfig { socket_mode: 0o660, libvirt_uri: default_libvirt_uri(), network_filter: None, + reconcile_interval_secs: default_reconcile_interval(), + collect_unattributed: false, } } } diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index f645a94e8..cfc9a29d6 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -94,6 +94,20 @@ enum NetdCommand { #[arg(long)] instance: Option, }, + /// Delete every interface netd holds for one VM. + /// + /// For a VM whose VMM will never ask again -- one whose directory was + /// deleted by hand, or whose instance is gone. A running VMM collects + /// these itself; this is for when there is no longer one to do it. + RemoveVm { + /// The `cvm.instance_id` of the VMM that created them. `netd list` + /// shows it. + #[arg(long)] + instance: String, + /// The VM's ID. + #[arg(long)] + vm: String, + }, } #[derive(ClapArgs)] @@ -218,8 +232,8 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< .await .context("failed to list netd interfaces")?; println!( - "{:<16} {:<8} {:<24} {:<38} {:>3} {}", - "INTERFACE", "KIND", "INSTANCE", "VM", "NIC", "FILTERED" + "{:<16} {:<8} {:<24} {:<38} {:>3} FILTERED", + "INTERFACE", "KIND", "INSTANCE", "VM", "NIC" ); let mut unattributed = 0; for record in &interfaces { @@ -238,7 +252,8 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< if record.bound { "yes" } else { "no" }, ); } - println!("\n{} interface(s)", interfaces.len()); + println!(); + println!("{} interface(s)", interfaces.len()); if unattributed > 0 { // Not a fault to fix by hand: an interface built before netd // recorded ownership, or by another netd, carries no record and @@ -249,6 +264,34 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< } Ok(()) } + NetdCommand::RemoveVm { instance, vm } => { + let removed = netd::remove_all(&config.socket, instance, vm) + .await + .context("failed to remove the VM's interfaces")?; + println!("removed {removed} interface(s) for {vm}"); + Ok(()) + } + } +} + +/// Collects host interfaces no VM claims, on an interval. +/// +/// The startup pass covers what a crash left behind. This covers what +/// accumulates while the VMM runs: a removal that raced a netd outage, a +/// teardown whose VM no longer exists to retry it. +async fn netd_reconcile_task(app: App) { + let interval_secs = app.config.netd.reconcile_interval_secs; + if interval_secs == 0 { + info!("periodic netd reconciliation is disabled"); + return; + } + let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + // The startup pass already ran, and this fires immediately on its first + // tick. + interval.tick().await; + loop { + interval.tick().await; + app.reconcile_netd_interfaces().await; } } @@ -416,8 +459,14 @@ async fn main() -> Result<()> { }; let state = app::App::new(config, supervisor); state.reload_vms().await.context("Failed to reload VMs")?; + // After the VMs are loaded, because the set of VMs this instance has is + // what the collection is decided against, and before the API is served, + // because a VM created between taking that set and acting on it would be + // in netd's listing and not in the set. + state.reconcile_netd_interfaces().await; tokio::spawn(auto_restart_task(state.clone())); tokio::spawn(log_rotation_task(state.clone())); + tokio::spawn(netd_reconcile_task(state.clone())); tokio::select! { result = run_external_api(state.clone(), figment.clone(), api_auth) => { diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index b5a7e004d..9b9b14944 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -58,6 +58,7 @@ const OPERATIONS: &[&str] = &[ "remove", "remove_all", "list", + "gc", "check", ]; /// The interface names netd may create. Reserved: anything matching it is @@ -70,6 +71,15 @@ const TAP_DIGEST_CHARS: usize = 12; const ALIAS_PREFIX: &str = "dstack1"; /// What the kernel stores in an interface alias, minus the terminator. const MAX_IFALIAS: usize = 255; +/// How long one sweep or collection may run before it reports what it has done +/// and stops. +/// +/// Under the operation lock and inside a serialized accept loop, so an +/// unbounded pass is not slow, it is an outage: one hung `virsh` per interface +/// would hold every other VM's prepare and remove behind it, and the caller +/// that asked has long since timed out. Partial progress reported honestly +/// beats total progress nobody is still waiting for. +const COLLECTION_DEADLINE: Duration = Duration::from_secs(20); #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InterfaceIdentity { @@ -151,6 +161,30 @@ pub struct IngressBinding { pub guest_port: u16, } +/// What a collection does about an interface it cannot attribute. +/// +/// Unattributed is not "nobody's". It is an interface built by a netd too old +/// to record ownership, by a third-party netd, or by this one in the instant +/// between creating an interface and recording it -- and on a host where two +/// VMM instances share a netd, one of those is another instance's running VM. +/// So the default is to leave it, and to say so. +/// +/// Nothing is stuck there: a collection still derives the names its own live +/// VMs would occupy and keeps those, so the interfaces of a running fleet +/// survive the upgrade that introduced the record, and each one gains a record +/// the next time its VM launches. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UnattributedPolicy { + /// Leave it and report it. + #[default] + Keep, + /// Delete it. Only safe where nothing else creates interfaces in netd's + /// name space -- a node with a single VMM instance -- and the operator + /// says so. + Remove, +} + /// One host resource netd holds. /// /// `instance_id` and `vm_id` are absent when the interface carries no record @@ -177,13 +211,6 @@ pub struct InterfaceRecord { pub bound: bool, } -impl InterfaceRecord { - /// Whether this is recorded as belonging to one VM of one VMM instance. - pub fn belongs_to(&self, instance_id: &str, vm_id: &str) -> bool { - self.instance_id.as_deref() == Some(instance_id) && self.vm_id.as_deref() == Some(vm_id) - } -} - /// What a netd can do, asked rather than inferred from a failure. /// /// `queues` and `ingress` report a *missing feature* by leaving a response @@ -287,6 +314,25 @@ pub enum Request { #[serde(default)] instance_id: String, }, + /// Delete every interface this VMM instance holds for a VM it no longer + /// has, and say what was left alone. + /// + /// The one operation that does not need to be told what to look for. Per-VM + /// teardown reaches only what its caller can still name; this reaches + /// what nothing names any more, which is the only place a leak can end up. + Gc { + instance_id: String, + /// Every VM this instance still has, running or not. Not the running + /// set: a VMM restarts under VMs that keep running, and collecting by + /// what is running would delete their interfaces out from under them. + live_vm_ids: Vec, + /// What to do with an interface carrying no ownership record. + #[serde(default)] + unattributed: UnattributedPolicy, + /// Report what would be collected without collecting it. + #[serde(default)] + dry_run: bool, + }, /// Verify a deterministic TAP and binding for operations and integration /// diagnostics. The VMM startup path uses Prepare rather than Check. Check { @@ -321,6 +367,10 @@ struct Response { /// the same reading `queues` and `ingress` get. #[serde(default, skip_serializing_if = "Option::is_none")] capabilities: Option, + /// Whether the pass stopped on its deadline with work left. See + /// [`COLLECTION_DEADLINE`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + incomplete: Option, /// Everything netd holds. Absent, rather than empty, from a netd that /// cannot enumerate: "I hold nothing" and "I cannot say" are answers a /// collection must not confuse. @@ -345,7 +395,14 @@ enum Outcome { ingress: Option>, }, /// A sweep names no single interface, so it reports how many it deleted. - Swept { removed: usize }, + Swept { removed: usize, incomplete: bool }, + /// A collection reports what it took as well as how much, because what it + /// took is the part an operator has to be able to disagree with. + Collected { + collected: Vec, + removed: usize, + incomplete: bool, + }, Hello(Capabilities), Listed(Vec), } @@ -369,6 +426,7 @@ impl Outcome { ingress: None, removed: None, capabilities: None, + incomplete: None, interfaces: None, error: None, }; @@ -384,7 +442,22 @@ impl Outcome { response.queues = queues; response.ingress = ingress; } - Self::Swept { removed } => response.removed = Some(removed), + Self::Swept { + removed, + incomplete, + } => { + response.removed = Some(removed); + response.incomplete = Some(incomplete); + } + Self::Collected { + collected, + removed, + incomplete, + } => { + response.removed = Some(removed); + response.incomplete = Some(incomplete); + response.interfaces = Some(collected); + } Self::Hello(capabilities) => response.capabilities = Some(capabilities), Self::Listed(interfaces) => response.interfaces = Some(interfaces), } @@ -536,6 +609,39 @@ pub async fn list(socket: &Path, instance_id: &str) -> Result, + pub incomplete: bool, +} + +/// Deletes every interface this VMM instance holds for a VM it no longer has. +/// See [`Request::Gc`]. +pub async fn collect( + socket: &Path, + instance_id: &str, + live_vm_ids: Vec, + unattributed: UnattributedPolicy, + dry_run: bool, +) -> Result { + let request = Request::Gc { + instance_id: instance_id.to_string(), + live_vm_ids, + unattributed, + dry_run, + }; + let response = exchange(socket, &request).await?; + Ok(Collection { + removed: response + .removed + .context("netd answered a collection without saying what it removed")?, + collected: response.interfaces.unwrap_or_default(), + incomplete: response.incomplete.unwrap_or_default(), + }) +} + /// This node's netd, as far as it can be asked. #[derive(Debug, Clone)] pub enum Reachability { @@ -622,6 +728,7 @@ async fn exchange(socket: &Path, request: &Request) -> Result { Request::Remove { .. } => "remove", Request::RemoveAll { .. } => "remove_all", Request::List { .. } => "list", + Request::Gc { .. } => "gc", Request::Check { .. } => "check", }; let exchange = async { @@ -746,6 +853,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul ingress: None, removed: None, capabilities: None, + incomplete: None, interfaces: None, error: Some(format!("{error:#}")), } @@ -811,9 +919,24 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { &instance_id, ))), Request::RemoveAll { instance_id, vm_id } => { - let removed = sweep_vm_interfaces(libvirt_uri, &instance_id, &vm_id)?; - Ok(Outcome::Swept { removed }) + let (removed, incomplete) = sweep_vm_interfaces(libvirt_uri, &instance_id, &vm_id)?; + Ok(Outcome::Swept { + removed, + incomplete, + }) } + Request::Gc { + instance_id, + live_vm_ids, + unattributed, + dry_run, + } => collect_garbage( + libvirt_uri, + &instance_id, + live_vm_ids, + unattributed, + dry_run, + ), Request::Remove { identity, filtered } => { validate_identity(&identity)?; let tap = tap_name(&identity); @@ -1020,6 +1143,26 @@ enum BindingCleanup { /// be running at all, and a stale binding left by an earlier, filtered /// interface at this name is still worth clearing when it is. BestEffort, + /// The caller has already decided about the binding. Used by a pass over + /// many interfaces, which asks libvirt once about all of them rather than + /// once per interface. + Skip, +} + +/// Removes one interface during a pass over many. +/// +/// A pass gets one answer about libvirt, not one per interface. Asking again +/// after it has failed is how a hung `libvirtd` turns a bounded collection into +/// an unbounded one: the deadline is reached having deleted nothing, while +/// every other VM on the host waits behind the operation lock. +fn remove_interface_in_pass(uri: &str, tap: &str, libvirt: &mut bool) -> Result<()> { + if *libvirt && !is_macvtap(tap) { + if let Err(error) = delete_binding(uri, tap) { + warn!(%tap, "could not clear a possible nwfilter binding: {error:#}"); + *libvirt = false; + } + } + remove_interface(uri, tap, BindingCleanup::Skip) } /// Deletes every interface a VM could hold, by deriving each name rather than @@ -1037,7 +1180,11 @@ enum BindingCleanup { /// less than the thing it replaced is not a sweep. Those names are decided /// against a single listing, because the whole point of enumerating a bounded /// space is that deciding one name stays cheap. -fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Result { +fn sweep_vm_interfaces( + libvirt_uri: &str, + instance_id: &str, + vm_id: &str, +) -> Result<(usize, bool)> { let identity = InterfaceIdentity { instance_id: instance_id.to_string(), vm_id: vm_id.to_string(), @@ -1045,21 +1192,31 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res }; validate_identity(&identity)?; let bindings = existing_bindings(libvirt_uri); + let mut libvirt = bindings.is_some(); let mut removed = 0; + let mut incomplete = false; let mut first_error = None; + let deadline = std::time::Instant::now() + COLLECTION_DEADLINE; for nic_index in 0..=MAX_NIC_INDEX { + if std::time::Instant::now() >= deadline { + warn!(%vm_id, nic_index, "sweep stopped on its deadline"); + incomplete = true; + break; + } let tap = tap_name(&InterfaceIdentity { nic_index, ..identity.clone() }); let present = Path::new("/sys/class/net").join(&tap).exists(); if !present { - if bindings - .as_ref() - .is_some_and(|bindings| bindings.contains(&tap)) + if libvirt + && bindings + .as_ref() + .is_some_and(|bindings| bindings.contains(&tap)) { if let Err(error) = delete_binding(libvirt_uri, &tap) { warn!(%tap, %error, "failed to remove orphaned nwfilter binding"); + libvirt = false; first_error.get_or_insert(error); } else { info!(%tap, %vm_id, "removed orphaned nwfilter binding"); @@ -1067,7 +1224,7 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res } continue; } - match remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort) { + match remove_interface_in_pass(libvirt_uri, &tap, &mut libvirt) { // Keep going after a failure. Stopping at the first one would leave // the rest of a VM's interfaces behind over one that is stuck. Err(error) => { @@ -1082,8 +1239,148 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res } match first_error { Some(error) => Err(error).context("failed to remove every interface for this VM"), - None => Ok(removed), + None => Ok((removed, incomplete)), + } +} + +/// What a collection would take, decided against a listing rather than against +/// the host. +/// +/// Split out so it can be reasoned about and tested without a privileged +/// daemon: the decision is the dangerous part, not the `ip link delete` that +/// follows it. +fn gc_plan( + records: &[InterfaceRecord], + instance_id: &str, + live_vm_ids: &HashSet, + live_names: &dyn Fn() -> HashSet, + policy: UnattributedPolicy, +) -> Vec { + let mut derived = None; + let mut collected = Vec::new(); + for record in records { + let keep = match (&record.instance_id, &record.vm_id) { + // Someone else's, and on a host where two VMM instances share one + // netd, "someone else's" includes a running VM. The record is the + // only thing that can tell them apart, which is why there is one. + (Some(owner), _) if owner != instance_id => true, + (Some(_), Some(vm_id)) => live_vm_ids.contains(vm_id), + // Cannot be attributed. Derive the names this instance's live VMs + // would occupy and keep those, so a fleet that was running before + // ownership was recorded survives the upgrade that introduced it. + _ => { + let names = derived.get_or_insert_with(live_names); + names.contains(&record.tap) || policy == UnattributedPolicy::Keep + } + }; + if !keep { + collected.push(record.clone()); + } + } + collected +} + +/// Every interface name this instance's live VMs could occupy. +/// +/// Only ever needed when something on the host carries no ownership record, so +/// it is derived lazily: |live| x 256 digests is cheap next to a `virsh` call +/// but not next to nothing. +fn live_interface_names(instance_id: &str, live_vm_ids: &HashSet) -> HashSet { + let mut names = HashSet::with_capacity(live_vm_ids.len() * (MAX_NIC_INDEX + 1)); + for vm_id in live_vm_ids { + for nic_index in 0..=MAX_NIC_INDEX { + names.insert(tap_name(&InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: vm_id.clone(), + nic_index, + })); + } } + names +} + +/// Deletes what [`gc_plan`] decided against. See [`Request::Gc`]. +fn collect_garbage( + libvirt_uri: &str, + instance_id: &str, + live_vm_ids: Vec, + policy: UnattributedPolicy, + dry_run: bool, +) -> Result { + if instance_id.is_empty() || instance_id.len() > 128 || instance_id.contains(':') { + bail!("invalid instance ID"); + } + let live: HashSet = live_vm_ids.into_iter().collect(); + let records = list_interfaces(libvirt_uri, ""); + let unattributed = records + .iter() + .filter(|record| record.instance_id.is_none()) + .count(); + let collected = gc_plan( + &records, + instance_id, + &live, + &|| live_interface_names(instance_id, &live), + policy, + ); + if dry_run { + info!( + %instance_id, + held = records.len(), + unattributed, + would_remove = collected.len(), + "collection dry run" + ); + return Ok(Outcome::Collected { + removed: 0, + incomplete: false, + collected, + }); + } + let mut libvirt = true; + let mut removed = 0; + let mut incomplete = false; + let mut taken = Vec::new(); + let deadline = std::time::Instant::now() + COLLECTION_DEADLINE; + for record in collected { + if std::time::Instant::now() >= deadline { + warn!("collection stopped on its deadline"); + incomplete = true; + break; + } + let outcome = if record.kind == "binding" { + // No interface left to delete, only the binding that outlived it. + delete_binding(libvirt_uri, &record.tap) + } else { + remove_interface_in_pass(libvirt_uri, &record.tap, &mut libvirt) + }; + match outcome { + Ok(()) => { + info!( + tap = %record.tap, + vm_id = record.vm_id.as_deref().unwrap_or("-"), + "collected interface no live VM claims" + ); + removed += 1; + taken.push(record); + } + // Keep going: one stuck interface must not shelter the rest. + Err(error) => warn!(tap = %record.tap, "failed to collect: {error:#}"), + } + } + if unattributed > 0 && policy == UnattributedPolicy::Keep { + info!( + %instance_id, + unattributed, + "interfaces carry no ownership record and were left alone; they gain one when their \ + VM next launches" + ); + } + Ok(Outcome::Collected { + removed, + incomplete, + collected: taken, + }) } /// Every resource netd owns, read off the host rather than out of a record. @@ -1171,6 +1468,7 @@ fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Re // outlives the interface. if !macvtap { match cleanup { + BindingCleanup::Skip => {} BindingCleanup::Required => delete_binding(libvirt_uri, tap)?, BindingCleanup::BestEffort => { // netd refuses to start without virsh, so the binary is always @@ -1657,7 +1955,9 @@ pub(crate) mod testing { response } "remove" | "check" => json!({"ok": true, "tap": "dtdeadbeef00"}), - "remove_all" => json!({"ok": true, "removed": 0}), + "remove_all" => json!({"ok": true, "removed": 0, "incomplete": false}), + "gc" => json!({"ok": true, "removed": 0, "incomplete": false, "interfaces": []}), + "list" => json!({"ok": true, "interfaces": []}), _ => json!({"ok": true}), } } @@ -2172,7 +2472,7 @@ mod tests { /// means nothing. #[test] fn a_sweep_reports_a_count_and_no_interface() { - let value = serde_json::to_value(Outcome::Swept { removed: 3 }.into_response()).unwrap(); + let value = serde_json::to_value(Outcome::Swept { removed: 3, incomplete: false }.into_response()).unwrap(); assert_eq!(value["removed"], 3); assert!(value.get("tap").is_none()); @@ -2252,6 +2552,101 @@ mod tests { assert!(tap_name(&identity("instance", "vm", 255)).len() < 16); } + fn record(tap: &str, instance: Option<&str>, vm: Option<&str>) -> InterfaceRecord { + InterfaceRecord { + tap: tap.to_string(), + kind: "tap".to_string(), + instance_id: instance.map(str::to_string), + vm_id: vm.map(str::to_string), + nic_index: instance.map(|_| 0), + bound: false, + } + } + + fn plan( + records: &[InterfaceRecord], + live: &[&str], + policy: UnattributedPolicy, + ) -> Vec { + let live: HashSet = live.iter().map(|vm| vm.to_string()).collect(); + gc_plan( + records, + "ours", + &live, + &|| live_interface_names("ours", &live), + policy, + ) + .into_iter() + .map(|record| record.tap) + .collect() + } + + /// The decision a collection is made of, which is the dangerous half: the + /// `ip link delete` that follows it is not the part that can take down a + /// running VM. + #[test] + fn a_collection_takes_only_what_this_instance_no_longer_claims() { + let records = [ + record("dt000000000001", Some("ours"), Some("live-vm")), + record("dt000000000002", Some("ours"), Some("dead-vm")), + // Another VMM instance on the same host. The record is the only + // thing that can tell this from ours, which is why there is one: + // without it a collection would delete another instance's running + // VM's networking. + record("dt000000000003", Some("theirs"), Some("dead-vm")), + ]; + assert_eq!( + plan(&records, &["live-vm"], UnattributedPolicy::Keep), + vec!["dt000000000002"] + ); + // A VM this instance no longer has at all is exactly the case per-VM + // teardown can never reach: nothing is left to name it. + assert_eq!( + plan(&records, &[], UnattributedPolicy::Keep).len(), + 2, + "both of ours, and never theirs" + ); + } + + /// The upgrade this has to survive: before ownership was recorded, every + /// interface on the host was unattributed, and a collection that deleted + /// them would take down the networking of every running VM at once. + #[test] + fn a_fleet_that_predates_the_ownership_record_survives_the_first_collection() { + let live_tap = tap_name(&identity("ours", "live-vm", 0)); + let records = [ + record(&live_tap, None, None), + record("dt0000000000ff", None, None), + ]; + + // The conservative default leaves both, and says so. + assert!(plan(&records, &["live-vm"], UnattributedPolicy::Keep).is_empty()); + + // Even told to collect, a name a live VM would occupy is kept: the + // record is missing, but the name is still derivable from the identity, + // and that is proof enough to keep something. + assert_eq!( + plan(&records, &["live-vm"], UnattributedPolicy::Remove), + vec!["dt0000000000ff".to_string()] + ); + } + + /// A binding outlives the interface it was bound to, so it is the one piece + /// of state that can never carry a record, and the one a collection over + /// interfaces alone would always miss. + #[test] + fn an_orphaned_binding_is_collectable_but_not_by_default() { + let mut binding = record("dt00000000beef", None, None); + binding.kind = "binding".to_string(); + binding.bound = true; + let records = [binding]; + assert!(plan(&records, &[], UnattributedPolicy::Keep).is_empty()); + assert_eq!( + plan(&records, &[], UnattributedPolicy::Remove), + vec!["dt00000000beef".to_string()] + ); + } + /// A netd that answers a sweep with no count did not sweep. Reading the /// absent field as zero is the same conflation `queues` and `ingress` are /// shaped to avoid, and here it would report a netd that cannot collect a diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 6ff5a9e0d..e89c30a6f 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -150,6 +150,17 @@ parameters = {} # authorize clients. [netd] socket = "/run/dstack/netd.sock" +# How often the VMM asks netd to collect interfaces no VM of its claims. The +# pass at startup is not optional and does not read this: it is the one moment +# the VMM knows every VM it has and everything a crash left behind is still +# there. This is the backstop for what accumulates while it runs. 0 disables it. +reconcile_interval_secs = 3600 +# Whether a collection may delete interfaces carrying no ownership record. An +# unattributed interface is not nobody's: where two VMM instances share one +# netd it may be the other one's running VM, and before netd recorded ownership +# every interface looked like this. Turn it on only on a node with a single VMM +# instance, where nothing else creates interfaces in netd's name space. +collect_unattributed = false # Applied when netd creates the socket itself. A systemd socket unit controls # its own SocketMode instead. socket_mode = 0o660 From dbc7d774af500cddc1972f021711704511e6ca42 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 02:23:56 -0700 Subject: [PATCH 07/34] feat(vmm): report which ports a VM actually publishes A port mapping is a request. Which NIC carries it decides who answers it: QEMU's `hostfwd` on a user-mode NIC, always; the node's netd on a bridge NIC, which -- like the netd in this repository -- may not forward host ports at all. `GetInfo` reported the request either way, so a VM could list published ports that nothing on the host forwarded any traffic to. So keep the answer. A prepare's `ingress` response is recorded on the NIC beside `netd_interface` and `device`, and `PortMapping.published` reports it: absent for a VM that is not running or a VMM that predates the field, true only where something actually publishes the port. Held to it per mapping, not per interface -- a netd may refuse one port out of a set, and the mapping that lost is the one worth naming. And refuse at deployment what a launch can only warn about. Two ways to have nowhere to go: a mapping that resolves to no NIC at all, and one that resolves to a bridge NIC on a node whose netd does not forward. Refusing costs nothing there, because nothing is running on the answer yet; refusing at launch would turn a VM that has been running with its ports dropped into an outage on upgrade. An update refuses only when it moved the ports or the networking, so a VM deployed before the node could answer for its ports stays editable in every other respect. The ownership contract that makes this collectable is now written down: a host port a netd publishes is owned by the interface and released with it, and there is deliberately no operation that releases one separately. A reservation outliving its interface could never be attributed to a VM again, because the interface is the only thing that carries a record. --- dstack/vmm/rpc/proto/vmm_rpc.proto | 8 +++ dstack/vmm/src/app.rs | 15 ++++-- dstack/vmm/src/app/network.rs | 1 + dstack/vmm/src/app/vm_info.rs | 86 +++++++++++++++++++++++++++++- dstack/vmm/src/config.rs | 9 ++++ dstack/vmm/src/main_service.rs | 63 +++++++++++++++++++++- dstack/vmm/src/netd.rs | 86 +++++++++++++++++++++--------- 7 files changed, 234 insertions(+), 34 deletions(-) diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 239ac2d79..72fa2fb87 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -191,6 +191,14 @@ message PortMapping { // user-mode NIC for management -- the topology multi-NIC was added for -- // put every published port on the management NIC. optional uint32 nic_index = 5; + // Whether the host port is actually reachable at the guest right now. + // + // Absent for a VM that is not running, and from a VMM that predates the + // field. A mapping was only ever a request: on a bridge NIC it is met by the + // node's netd, which may not forward host ports at all, and reporting the + // request as though it were the answer is how published ports could be + // listed for a VM nothing on the host forwarded any traffic to. + optional bool published = 6; } // Partial configuration used when mutating an existing VM. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 9553750ed..633d40c44 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -60,7 +60,7 @@ mod host_share; mod id_pool; mod image; mod mr_config; -mod network; +pub(crate) mod network; mod qemu; pub(crate) mod registry; mod vm_info; @@ -806,6 +806,9 @@ impl App { // handed out is its own to state -- and the mapping that lost // is the one worth naming. Some(bound) => { + // What was answered, not what was asked. `GetInfo` reports + // the difference rather than the request. + network.ingress = bound.clone(); for request in asked { if !bound.iter().any(|binding| { binding.protocol == request.protocol @@ -1036,7 +1039,10 @@ impl App { debug!(vm_id, %error, "no netd to release interfaces from") } Err(error) => { - warn!(vm_id, "failed to release netd-managed interfaces: {error:#}") + warn!( + vm_id, + "failed to release netd-managed interfaces: {error:#}" + ) } } return; @@ -2416,9 +2422,8 @@ mod tests { "an operation it does not have is not sent" ); - let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ - "hello", "gc", - ])); + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&["hello", "gc"])); let app = app_talking_to(netd.socket()); app.reconcile_netd_interfaces().await; let request = &netd.seen()[1]; diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index 05fb6e42e..db5d00792 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -41,6 +41,7 @@ pub(crate) fn resolve_networking( // node configuration that names either is rejected at startup. resolved.netd_interface = crate::config::NetdInterface::None; resolved.device.clear(); + resolved.ingress.clear(); if !networking.bridge.is_empty() { resolved.nic.bridge = networking.bridge.clone(); } diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 48af6f864..4d955bb64 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -126,6 +126,31 @@ fn sanitize_optional>(value: Option) -> Option { value.filter(|value| !value.as_ref().trim().is_empty()) } +/// Whether one mapping's host port is actually reachable at the guest. +/// +/// A mapping is a request, and which NIC carries it decides who answers it: +/// QEMU's `hostfwd` on a user-mode NIC, always; the node's netd on a bridge +/// NIC, which may not forward host ports at all. A mapping naming a NIC the VM +/// no longer has is answered by nobody. Reporting the request as the answer is +/// how a VM could list published ports that nothing on the host forwarded. +fn published_at(mapping: &crate::app::PortMapping, networks: &[Networking]) -> bool { + let Some(nic_index) = crate::app::network::ingress_nic(mapping, networks) else { + return false; + }; + let Some(network) = networks.get(nic_index) else { + return false; + }; + if network.nic.mode == crate::config::NetworkingMode::User { + // QEMU carries these itself, for as long as it is up. + return true; + } + network.ingress.iter().any(|binding| { + binding.protocol == mapping.protocol.as_str() + && binding.host_port == mapping.from + && binding.guest_port == mapping.to + }) +} + impl VmInfo { /// Takes no `CvmConfig` on purpose. Everything it reports about a VM's /// data plane was decided when that VM launched and written into @@ -198,6 +223,9 @@ impl VmInfo { .port_map .iter() .map(|mapping| pb::PortMapping { + published: self + .running + .then(|| published_at(mapping, effective_networks)), nic_index: mapping.nic_index.map(|index| index as u32), protocol: mapping.protocol.as_str().into(), host_address: mapping.address.to_string(), @@ -328,8 +356,62 @@ impl VmState { #[cfg(test)] mod tests { - use super::{interfaces_to_proto, networking_to_proto, sanitize_optional}; - use crate::config::{NetworkingMode, NicNetworking}; + use super::{interfaces_to_proto, networking_to_proto, published_at, sanitize_optional}; + use crate::app::PortMapping; + use crate::config::{Networking, NetworkingMode, NicNetworking, Protocol}; + use crate::netd::IngressBinding; + + fn nic(mode: NetworkingMode) -> Networking { + Networking { + nic: NicNetworking { + mode, + ..NicNetworking::default() + }, + ..Networking::default() + } + } + + fn mapping(host_port: u16, nic_index: Option) -> PortMapping { + PortMapping { + address: "0.0.0.0".parse().unwrap(), + protocol: Protocol::Tcp, + from: host_port, + to: 8080, + nic_index, + } + } + + /// A mapping is a request. Which NIC carries it decides who answers it, + /// and on a bridge the answer comes from a netd that may not forward host + /// ports at all -- so reporting the request as the answer is how a VM + /// could list published ports nothing on the host forwarded. + #[test] + fn a_port_is_reported_published_only_where_something_publishes_it() { + // QEMU carries a user-mode NIC's mappings itself. + let networks = [nic(NetworkingMode::User)]; + assert!(published_at(&mapping(443, None), &networks)); + + // A bridge NIC's are netd's to publish, and this node's netd said + // nothing about them. + let networks = [nic(NetworkingMode::Bridge)]; + assert!(!published_at(&mapping(443, None), &networks)); + + // Until it does. + let mut published = nic(NetworkingMode::Bridge); + published.ingress = vec![IngressBinding { + protocol: "tcp".into(), + host_address: "0.0.0.0".into(), + host_port: 443, + guest_port: 8080, + }]; + let networks = [published]; + assert!(published_at(&mapping(443, None), &networks)); + // Answered for one port is not answered for another. + assert!(!published_at(&mapping(444, None), &networks)); + + // A mapping naming a NIC the VM no longer has is answered by nobody. + assert!(!published_at(&mapping(443, Some(7)), &networks)); + } /// Custom mode hands the operator the whole netdev string and the VMM never /// parses it, so it has no data-plane state to report. Reporting the diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index f6fe7962b..89695cf9e 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -1095,6 +1095,15 @@ pub struct Networking { /// the VM runs, and what has to be removed is what was created. #[serde(default, skip_serializing_if = "NetdInterface::is_none")] pub netd_interface: NetdInterface, + /// The host ports netd actually published for this NIC. + /// + /// Runtime state, like `device`: resolution always clears it. What was + /// asked for lives in the manifest; this is what was answered, and the + /// difference between the two is the whole reason to keep it. Reporting + /// the request as though it were the answer is how a VM's ports could be + /// listed as published while nothing on the host forwarded them. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub ingress: Vec, } /// The host interface netd created for a NIC, if any. diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 98f113c08..15cfba99c 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -24,8 +24,10 @@ use path_absolutize::Absolutize; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; +use crate::app::network::ingress_nic; use crate::app::{ - mode_carries_ingress, needs_swtpm, resolve_networking, validate_resolved_network, + mode_carries_ingress, needs_swtpm, resolve_networking, resolved_networks, + validate_resolved_network, validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, }; @@ -831,12 +833,63 @@ impl RpcHandler { Ok(true) } + + /// Refuses a port mapping this node has no way to publish. + /// + /// At deployment, where refusing costs nothing: nothing is running on the + /// answer yet, and the alternative is a VM that reports published ports + /// nothing on the host forwards. A launch only warns about the same thing, + /// because a VM deployed before this has been running with its ports + /// dropped, and failing it on upgrade would turn a silent misconfiguration + /// into an outage. + /// + /// Two ways to have nowhere to go. A mapping can resolve to no NIC at all + /// -- a VM whose every NIC is macvtap or custom -- or to a bridge NIC on a + /// node whose netd builds interfaces and does not forward host ports, which + /// is what the netd in this repository does. + async fn refuse_unpublishable_ports(&self, manifest: &Manifest) -> Result<()> { + let networks = resolved_networks(manifest, &self.app.config.cvm); + let mut needs_netd = Vec::new(); + for mapping in &manifest.port_map { + let backend = ingress_nic(mapping, &networks).and_then(|index| networks.get(index)); + let named = format!( + "{} {}:{}", + mapping.protocol.as_str(), + mapping.address, + mapping.from + ); + match backend { + None => bail!( + "port mapping {named} has no NIC to enter through: this VM has no user-mode \ + or bridge interface" + ), + // QEMU carries these itself. + Some(network) if network.nic.mode == NetworkingMode::User => {} + Some(_) => needs_netd.push(named), + } + } + if needs_netd.is_empty() { + return Ok(()); + } + let reachability = self.app.netd_capabilities().await; + if reachability.forwards_ingress() { + return Ok(()); + } + bail!( + "port mapping {} enters through a bridge NIC, which only netd can publish, and the \ + netd on this node does not forward host ports ({}). Put the mapping on a user-mode \ + NIC with @, or deploy a netd that forwards", + needs_netd.join(", "), + reachability.describe() + ) + } } impl VmmRpc for RpcHandler { async fn create_vm(self, request: VmConfiguration) -> Result { let manifest = create_manifest_from_vm_config(request.clone(), &self.app.config.cvm)?; self.validate_port_mapping_conflicts(None, &manifest.port_map)?; + self.refuse_unpublishable_ports(&manifest).await?; let id = manifest.id.clone(); info!(vm_id = %id, "create_vm RPC called"); let app_id = manifest.app_id.clone(); @@ -1032,6 +1085,14 @@ impl VmmRpc for RpcHandler { &manifest.port_map, &resolved_nic_modes(&manifest.networks, &self.app.config.cvm, manifest.vcpu), )?; + // Only when this request moved one of them. A VM deployed before the + // node could answer for its ports must stay editable in every other + // respect: read-modify-write sends the whole configuration back, and + // refusing a memory change over a port mapping nobody touched would + // make the VM unmanageable rather than fixed. + if request.update_ports || request.update_networking { + self.refuse_unpublishable_ports(&manifest).await?; + } let compose_file = fs::read_to_string(vm_work_dir.app_compose_path()) .context("failed to read app compose for swtpm decision")?; manifest.swtpm = needs_swtpm( diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 9b9b14944..7c1682db9 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -120,6 +120,15 @@ pub struct PrepareBridgeRequest { /// sends. Whether a netd forwards them is its own business; this states the /// requirement rather than assuming it is met, and the response says what /// was actually done. + /// + /// Owned by the interface. Whatever a netd establishes to satisfy this is + /// released when the interface is -- by `remove`, by `remove_all`, or by a + /// collection -- and there is deliberately no operation that releases it + /// separately. A host port outliving the interface it was forwarding to is + /// a leak nothing would ever collect: the interface is the only thing that + /// carries an ownership record, so a reservation that survived it could + /// never be attributed to a VM again. A prepare for an identity that + /// already has an interface replaces both together. #[serde(default)] pub ingress: Vec, } @@ -280,6 +289,9 @@ pub enum Request { Hello, PrepareBridge(PrepareBridgeRequest), PrepareMacvtap(PrepareMacvtapRequest), + /// + /// Releases everything the interface owns, including any host ports a + /// forwarding netd published for it. See [`PrepareBridgeRequest::ingress`]. Remove { #[serde(flatten)] identity: InterfaceIdentity, @@ -296,6 +308,9 @@ pub enum Request { /// that lost a NIC leaves the same thing behind. Both are found here /// without a record, because every name netd can produce for a VM is /// derivable from its identity. + /// + /// Releases everything those interfaces own, host ports included. See + /// [`PrepareBridgeRequest::ingress`]. RemoveAll { instance_id: String, vm_id: String, @@ -320,6 +335,9 @@ pub enum Request { /// The one operation that does not need to be told what to look for. Per-VM /// teardown reaches only what its caller can still name; this reaches /// what nothing names any more, which is the only place a leak can end up. + /// + /// Releases everything those interfaces own, host ports included. See + /// [`PrepareBridgeRequest::ingress`]. Gc { instance_id: String, /// Every VM this instance still has, running or not. Not the running @@ -395,7 +413,10 @@ enum Outcome { ingress: Option>, }, /// A sweep names no single interface, so it reports how many it deleted. - Swept { removed: usize, incomplete: bool }, + Swept { + removed: usize, + incomplete: bool, + }, /// A collection reports what it took as well as how much, because what it /// took is the part an operator has to be able to disagree with. Collected { @@ -471,7 +492,10 @@ pub fn tap_name(identity: &InterfaceIdentity) -> String { identity.instance_id, identity.vm_id, identity.nic_index ); let digest = Sha256::digest(input.as_bytes()); - format!("{TAP_PREFIX}{}", hex::encode(&digest[..TAP_DIGEST_CHARS / 2])) + format!( + "{TAP_PREFIX}{}", + hex::encode(&digest[..TAP_DIGEST_CHARS / 2]) + ) } /// The ownership record netd writes onto every interface it creates. @@ -679,7 +703,11 @@ impl Reachability { Self::Unreachable => "unreachable".to_string(), Self::Legacy => "reachable, predates capability reporting".to_string(), Self::Capable(capabilities) => { - format!("{} [{}]", capabilities.version, capabilities.operations.join(" ")) + format!( + "{} [{}]", + capabilities.version, + capabilities.operations.join(" ") + ) } } } @@ -914,10 +942,9 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { Request::PrepareMacvtap(request) => { prepare_macvtap(libvirt_uri, &request, config.filter_policy()) } - Request::List { instance_id } => Ok(Outcome::Listed(list_interfaces( - libvirt_uri, - &instance_id, - ))), + Request::List { instance_id } => { + Ok(Outcome::Listed(list_interfaces(libvirt_uri, &instance_id))) + } Request::RemoveAll { instance_id, vm_id } => { let (removed, incomplete) = sweep_vm_interfaces(libvirt_uri, &instance_id, &vm_id)?; Ok(Outcome::Swept { @@ -1180,11 +1207,7 @@ fn remove_interface_in_pass(uri: &str, tap: &str, libvirt: &mut bool) -> Result< /// less than the thing it replaced is not a sweep. Those names are decided /// against a single listing, because the whole point of enumerating a bounded /// space is that deciding one name stays cheap. -fn sweep_vm_interfaces( - libvirt_uri: &str, - instance_id: &str, - vm_id: &str, -) -> Result<(usize, bool)> { +fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Result<(usize, bool)> { let identity = InterfaceIdentity { instance_id: instance_id.to_string(), vm_id: vm_id.to_string(), @@ -1412,8 +1435,9 @@ fn list_interfaces(libvirt_uri: &str, instance_id: &str) -> Vec // creates. Listing it would invite a collection to delete it. continue; }; - let alias = std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifalias")) - .unwrap_or_default(); + let alias = + std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifalias")) + .unwrap_or_default(); let owner = owner_of(&tap, &alias); seen.insert(tap.clone()); records.push(InterfaceRecord { @@ -1877,7 +1901,9 @@ pub(crate) mod testing { }; recorder.lock().expect("poisoned").push(request.clone()); let response = answer(&behavior, &request); - let _ = stream.write_all(&serde_json::to_vec(&response).unwrap()).await; + let _ = stream + .write_all(&serde_json::to_vec(&response).unwrap()) + .await; let _ = stream.shutdown().await; }); } @@ -2472,13 +2498,19 @@ mod tests { /// means nothing. #[test] fn a_sweep_reports_a_count_and_no_interface() { - let value = serde_json::to_value(Outcome::Swept { removed: 3, incomplete: false }.into_response()).unwrap(); + let value = serde_json::to_value( + Outcome::Swept { + removed: 3, + incomplete: false, + } + .into_response(), + ) + .unwrap(); assert_eq!(value["removed"], 3); assert!(value.get("tap").is_none()); // A prepare still names one, because the caller hands that name to QEMU. - let value = - serde_json::to_value(Outcome::tap("dtabc".into()).into_response()).unwrap(); + let value = serde_json::to_value(Outcome::tap("dtabc".into()).into_response()).unwrap(); assert_eq!(value["tap"], "dtabc"); assert!(value.get("removed").is_none()); } @@ -2531,7 +2563,9 @@ mod tests { assert!(validate_identity(&identity("instance", &long, 0)).is_ok()); let identity_too_long = identity(&"i".repeat(128), &long, 255); assert!(interface_alias(&identity_too_long).len() > MAX_IFALIAS); - let error = validate_identity(&identity_too_long).unwrap_err().to_string(); + let error = validate_identity(&identity_too_long) + .unwrap_err() + .to_string(); assert!(error.contains("too long to record"), "{error}"); } @@ -2563,11 +2597,7 @@ mod tests { } } - fn plan( - records: &[InterfaceRecord], - live: &[&str], - policy: UnattributedPolicy, - ) -> Vec { + fn plan(records: &[InterfaceRecord], live: &[&str], policy: UnattributedPolicy) -> Vec { let live: HashSet = live.iter().map(|vm| vm.to_string()).collect(); gc_plan( records, @@ -2661,7 +2691,10 @@ mod tests { assert!(!is_unreachable(&error)); let netd = testing::FakeNetd::spawn(testing::Behavior::capable(&["hello", "remove_all"])); - assert_eq!(remove_all(netd.socket(), "instance", "vm").await.unwrap(), 0); + assert_eq!( + remove_all(netd.socket(), "instance", "vm").await.unwrap(), + 0 + ); } /// Every "an unreachable netd is not a failure" branch in the VMM hangs @@ -2696,7 +2729,8 @@ mod tests { assert!(!reachability.supports("remove_all")); assert!(!reachability.forwards_ingress()); - let netd = testing::FakeNetd::spawn(testing::Behavior::forwarding(&["hello", "remove_all"])); + let netd = + testing::FakeNetd::spawn(testing::Behavior::forwarding(&["hello", "remove_all"])); let reachability = probe(netd.socket()).await; assert!(reachability.supports("remove_all")); assert!(!reachability.supports("gc")); From cf315020fa9b8d662860a62b9aeea562af3860bc Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 02:28:14 -0700 Subject: [PATCH 08/34] test(netd): put the ownership record next to the kernel, and document it Everything else here reasons about strings. This creates a real TAP, reads the alias back off it, finds it by enumeration, and checks that the guards refuse what they are meant to: a device with one of netd's names that netd did not create, and a record that does not re-derive the name it is written on. It refuses to run in the host's network namespace. Unsharing one from inside the test is not enough -- `/sys/class/net` keeps showing the old namespace until sysfs is remounted, which is most of what `ip netns exec` does -- so it asks to be put in one instead, and says so in the doc comment. That also means it can never touch a real node's interfaces, including when it fails. The docs gain what an operator needs to act on any of this: how to read an interface's owner, how a collection decides, why an interface with no ownership record is kept rather than deleted, and why upgrading needs no migration step. --- docs/bridge-networking.md | 70 ++++++++++++++++++++++++ docs/libvirt-network-filter.md | 3 ++ dstack/vmm/src/main_service.rs | 5 +- dstack/vmm/src/netd.rs | 98 ++++++++++++++++++++++++++++++++++ 4 files changed, 173 insertions(+), 3 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 4c763fc2f..06a79abb3 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -173,6 +173,7 @@ needs an `allow` line for the bridge. - Guest MAC address is derived from SHA256 of the VM ID, with an optional configurable prefix (stable across restarts for DHCP IP consistency) - The host DHCP server (dnsmasq) assigns an IP to the VM - The TAP outlives QEMU and is deleted when the VMM tears the VM's networking down, so a VM that crashes does not leave its filter rules attached to a name the next VM could take +- Every interface `netd` creates records which VM of which VMM instance it belongs to, in the kernel's interface alias — see [Who owns an interface](#who-owns-an-interface) - The VMM process needs neither root nor `CAP_NET_ADMIN`; `netd` holds that privilege in a separate service ### MAC address prefix @@ -225,6 +226,75 @@ A mapping resolves to exactly one NIC, and that NIC's backend decides the mechanism: `hostfwd=` for user mode, `netd` for a bridge. Nothing can be claimed by both. +### Which ports are actually published + +A port mapping is a *request*. Whether it is met depends on which NIC carries +it: QEMU publishes a user-mode NIC's mappings itself, while a bridge NIC's can +only be published by `netd`, and the `netd` in this repository builds +interfaces and does not forward host ports. + +`GetInfo` reports `published` per mapping so the difference is visible rather +than assumed. A deployment that asks for something this node cannot publish is +refused outright — nothing is running on the answer yet — while a VM deployed +before the node could answer only gets a warning at launch, so an upgrade never +turns a silent misconfiguration into an outage. + +To publish a bridge NIC's ports, run a `netd` that forwards. It reports +`ingress: true` in its `hello` and echoes what it established in each prepare; +the VMM records that answer and holds it to it per mapping. + +## Who owns an interface + +`netd` names an interface `dt<12 hex>`, a digest of (VMM instance, VM, NIC +index). That answers "where is this VM's interface" but not "whose is this +interface" — and the second question is the one a leaked interface poses. So +`netd` also records the identity on the interface itself: + +```console +$ ip -d link show dtc41d9e0b7a52 | grep alias + alias dstack1:0:path-3f9a1c8e7d2b4a60:0a1b2c3d4e5f6071 +``` + +The kernel holds that for exactly the interface's lifetime, so unlike a file on +disk it cannot be written late, lost, or left behind. It is a hint, never an +authority: a record is believed only when re-deriving the interface name from +it reproduces the name it is written on, so a forged, truncated or ambiguous +record reads the same as no record at all. + +```bash +# What netd holds on this host +sudo dstack-vmm netd list + +# Everything one VM holds, for a VM whose VMM will never ask again +sudo dstack-vmm netd remove-vm --instance path-3f9a1c8e7d2b4a60 --vm 0a1b2c3d4e5f6071 +``` + +### Collection + +The VMM asks `netd` to collect interfaces no VM of its claims: once at startup, +after it has loaded its VMs and before it serves its API, and then every +`netd.reconcile_interval_secs`. That is what reaches an interface no per-VM +teardown can — one whose VM was removed while the VMM was down, or whose +workdir was deleted by hand. + +Three rules: + +| What the interface is recorded as | What happens | +| --- | --- | +| Another VMM instance's | Never touched. Several VMM instances share one `netd`, and the record is the only thing that can tell that instance's *running* VM from garbage | +| This instance's, for a VM it no longer has | Collected | +| Nothing that checks out | Kept and reported. Set `netd.collect_unattributed = true` to collect it | + +An interface with no record is not nobody's: before `netd` recorded ownership +every interface looked like this, and on a host with two VMM instances one of +them may be the other's running VM. Turn `collect_unattributed` on only where a +single VMM instance owns the host. + +Upgrading is safe without touching anything: a collection also derives the +names its own live VMs would occupy and keeps those, so a fleet running from +before this existed survives the first pass, and each interface gains a record +the next time its VM launches. + ### Mixing networking modes Bridge and user-mode VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index 4ba827ec5..bf99a3ab0 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -20,6 +20,9 @@ The measurable acceptance criteria are: that TAP carries a binding. - `network_filter = "libvirt"` creates the TAP and filter binding before QEMU is submitted to Supervisor, and uses QEMU `-netdev tap`. +- An nwfilter binding outlives the TAP it was bound to, so teardown and + collection both clear a binding whose interface is already gone. `dstack-vmm + netd list` shows one as a `binding` row with no interface. - A failed TAP or filter setup prevents QEMU from starting and rolls back all interfaces prepared for that VM. - Normal stop and removal delete the filter binding and TAP. diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 15cfba99c..85866bc19 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -27,9 +27,8 @@ use tracing::{info, warn}; use crate::app::network::ingress_nic; use crate::app::{ mode_carries_ingress, needs_swtpm, resolve_networking, resolved_networks, - validate_resolved_network, - validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, - VmWorkDir, + validate_resolved_network, validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, + Manifest, PortMapping, VmWorkDir, }; use crate::config::{CvmConfig, Networking, NetworkingMode, NicNetworking}; diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 7c1682db9..67c4e2cb4 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -2677,6 +2677,104 @@ mod tests { ); } + /// Everything else here reasons about strings. This puts the reasoning + /// next to the kernel: that an alias survives on a device netd actually + /// creates, that enumeration finds it, that the guards refuse what they + /// are meant to, and that removal leaves nothing. + /// + /// Refuses to run in the host's network namespace, so it cannot touch a + /// real node's interfaces even when it fails. Unsharing one from inside + /// the test is not enough: `/sys/class/net` keeps showing the old + /// namespace until sysfs is remounted, which is most of what `ip netns + /// exec` does. So it asks to be put in one: + /// + /// ```text + /// cargo test -p dstack-vmm --bins --no-run + /// sudo ip netns add dstack-netd-test + /// sudo ip netns exec dstack-netd-test \ + /// target/debug/deps/dstack_vmm- --ignored --test-threads=1 + /// sudo ip netns del dstack-netd-test + /// ``` + #[test] + #[ignore = "needs root and its own network namespace; see the doc comment"] + fn a_real_interface_carries_its_record_and_removal_leaves_nothing() { + assert!( + nix::unistd::Uid::effective().is_root(), + "this test needs root" + ); + let (mine, init) = ( + std::fs::read_link("/proc/self/ns/net").unwrap(), + std::fs::read_link("/proc/1/ns/net").unwrap(), + ); + assert_ne!( + mine, init, + "run this inside its own network namespace; it creates and deletes interfaces" + ); + // Nothing in this namespace to talk to, which is also the state of a + // node that does not filter: the listing has to work without libvirt. + let uri = "qemu:///nonexistent-for-this-test"; + + let nic = identity("test-instance", "vm-1", 2); + let tap = tap_name(&nic); + ip(&["tuntap", "add", "dev", &tap, "mode", "tap"]).unwrap(); + set_alias(&tap, &nic).unwrap(); + assert!(is_tuntap(&tap), "the kernel publishes tun_flags for a TAP"); + + let records = list_interfaces(uri, ""); + let record = records + .iter() + .find(|record| record.tap == tap) + .expect("an interface netd created is one netd can find"); + assert_eq!(record.instance_id.as_deref(), Some("test-instance")); + assert_eq!(record.vm_id.as_deref(), Some("vm-1")); + assert_eq!(record.nic_index, Some(2)); + assert_eq!(record.kind, "tap"); + // Narrowing by instance is what keeps one VMM's collection off + // another's interfaces. + assert_eq!(list_interfaces(uri, "test-instance").len(), 1); + assert!(list_interfaces(uri, "someone-else").is_empty()); + + // A device with one of netd's names that netd did not create. The name + // is 48 bits of digest, so this is not about collisions -- it is that + // `ip link delete` does not ask what it is deleting, and netd runs as + // root. + let impostor = tap_name(&identity("test-instance", "not-a-tap", 0)); + ip(&["link", "add", &impostor, "type", "dummy"]).unwrap(); + assert!(is_managed_name(&impostor)); + assert!( + !list_interfaces(uri, "") + .iter() + .any(|record| record.tap == impostor), + "a device netd did not create is not offered up for collection" + ); + let refused = remove_interface(uri, &impostor, BindingCleanup::Skip).unwrap_err(); + assert!(refused.to_string().contains("refusing to delete")); + + // An interface whose record does not re-derive its own name proves + // nothing, and lands in the same bucket as no record at all. + ip(&[ + "link", + "set", + "dev", + &tap, + "alias", + "dstack1:2:test-instance:some-other-vm", + ]) + .unwrap(); + let records = list_interfaces(uri, ""); + let record = records.iter().find(|record| record.tap == tap).unwrap(); + assert!(record.instance_id.is_none(), "a forged record is no record"); + + remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); + assert!(!Path::new("/sys/class/net").join(&tap).exists()); + assert!(!list_interfaces(uri, "") + .iter() + .any(|record| record.tap == tap)); + // Removing what is not there is not an error: a sweep derives names + // and most of them miss. + remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); + } + /// A netd that answers a sweep with no count did not sweep. Reading the /// absent field as zero is the same conflation `queues` and `ingress` are /// shaped to avoid, and here it would report a netd that cannot collect a From c7557e340ed8432b30204599d7a70a1abc12ef55 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 02:30:51 -0700 Subject: [PATCH 09/34] fix(vmm): claim a VM's interfaces from its directory, not just from what loaded A VM whose manifest is corrupt or whose image is missing fails to load and is only logged -- but its QEMU may well still be running. Collecting by what loaded would delete a running VM's networking over a file the VMM could not parse. A directory is enough of a claim, and what has been removed for real leaves none behind. An unreadable VM directory is not an empty one either: read as empty it would offer every interface on the host up for collection, so it is an answer the reconciliation declines to act on at all. --- dstack/vmm/src/app.rs | 85 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 633d40c44..b0700e828 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -933,6 +933,40 @@ impl App { reachability } + /// Every VM whose interfaces this instance may still be using. + /// + /// Wider than the VMs it managed to load. A VM whose manifest is corrupt + /// or whose image is missing fails to load and is only logged -- but its + /// QEMU may well still be running, and collecting by what loaded would + /// delete a running VM's networking over a file the VMM could not parse. + /// A directory is enough of a claim: what has been removed for real leaves + /// none behind. + /// `None` when the answer cannot be established, which is not the same as + /// nobody claiming anything: an unreadable VM directory read as empty + /// would offer every interface on the host up for collection. + fn claimable_vm_ids(&self) -> Option> { + let mut ids: HashSet = self.lock().vms.keys().cloned().collect(); + match fs::read_dir(self.vm_dir()) { + Ok(entries) => { + for entry in entries.flatten() { + if entry.path().is_dir() { + if let Some(id) = entry.file_name().to_str() { + ids.insert(id.to_string()); + } + } + } + } + // A VMM that has never run has no directory and no VMs, which is a + // knowable answer rather than an unknowable one. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + warn!("failed to read the VM directory: {error}; not collecting interfaces"); + return None; + } + } + Some(ids.into_iter().collect()) + } + /// Deletes every host interface netd holds for a VM this instance no /// longer has. /// @@ -965,7 +999,9 @@ impl App { ); return; } - let live: Vec = self.lock().vms.keys().cloned().collect(); + let Some(live) = self.claimable_vm_ids() else { + return; + }; let policy = if self.config.netd.collect_unattributed { netd::UnattributedPolicy::Remove } else { @@ -2323,7 +2359,7 @@ mod tests { App::new(config, SupervisorClient::new("http://127.0.0.1:0")) } - fn app_talking_to(netd_socket: &Path) -> App { + fn test_config(netd_socket: &Path, run_path: &Path) -> Config { use rocket::figment::providers::Format as _; let mut config: Config = rocket::figment::Figment::from( rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), @@ -2332,7 +2368,50 @@ mod tests { .unwrap(); config.netd.socket = netd_socket.to_path_buf(); config.cvm.instance_id = "test-instance".to_string(); - App::new(config, SupervisorClient::new("http://127.0.0.1:0")) + config.run_path = run_path.to_path_buf(); + config + } + + fn app_talking_to(netd_socket: &Path) -> App { + let run_path = netd_socket.parent().unwrap_or(Path::new("/nonexistent")); + App::new( + test_config(netd_socket, run_path), + SupervisorClient::new("http://127.0.0.1:0"), + ) + } + + /// A VM whose manifest is corrupt or whose image is missing fails to load + /// and is only logged -- but its QEMU may well still be running. Collecting + /// by what loaded would delete a running VM's networking over a file the + /// VMM could not parse. + #[test] + fn a_vm_that_failed_to_load_still_claims_its_interfaces() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("vm-that-did-not-load")).unwrap(); + std::fs::write(dir.path().join("not-a-vm"), "").unwrap(); + let app = App::new( + test_config(Path::new("/nonexistent/netd.sock"), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + assert_eq!( + app.claimable_vm_ids(), + Some(vec!["vm-that-did-not-load".to_string()]) + ); + + // Unreadable is not "nobody claims anything", which would offer every + // interface on the host up for collection. + let app = App::new( + test_config( + Path::new("/nonexistent/netd.sock"), + Path::new("/proc/self/environ"), + ), + SupervisorClient::new("http://127.0.0.1:0"), + ); + assert_eq!(app.claimable_vm_ids(), None); + + // A VMM that has never run is a knowable answer, not an unknowable one. + let app = app_talking_to(Path::new("/nonexistent/netd.sock")); + assert_eq!(app.claimable_vm_ids(), Some(Vec::new())); } /// A netd outage must not become a fleet that cannot be stopped. From 6888ac401f47807c13195d8dbb278ae367baaa5a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 02:32:06 -0700 Subject: [PATCH 10/34] fix(vmm): refuse an instance ID no interface could be recorded as belonging to At startup rather than at the first launch. The VMM derives one that is always valid; an operator who configured their own learns here rather than from the first VM that fails to get a NIC. --- dstack/vmm/src/main.rs | 1 + dstack/vmm/src/netd.rs | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index cfc9a29d6..b926ba5c7 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -352,6 +352,7 @@ async fn main() -> Result<()> { // Preserve the existing startup validation. The broader static checks are // opt-in through `check-config` until they have seen wider deployment use. + netd::validate_instance_id(&config.cvm.instance_id)?; config .host_api .validate() diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 67c4e2cb4..7b8f13316 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -389,7 +389,8 @@ struct Response { /// [`COLLECTION_DEADLINE`]. #[serde(default, skip_serializing_if = "Option::is_none")] incomplete: Option, - /// Everything netd holds. Absent, rather than empty, from a netd that + /// For a listing, everything netd holds; for a collection, what it took, + /// or would take on a dry run. Absent, rather than empty, from a netd that /// cannot enumerate: "I hold nothing" and "I cannot say" are answers a /// collection must not confuse. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -549,6 +550,20 @@ pub fn is_managed_name(interface: &str) -> bool { .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } +/// Rejects an instance ID no interface could be recorded as belonging to. +/// +/// At startup rather than at the first launch. The VMM derives one that is +/// always valid; an operator who configured their own learns here rather than +/// from the first VM that fails to get a NIC. +pub fn validate_instance_id(instance_id: &str) -> Result<()> { + validate_identity(&InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: "0".repeat(64), + nic_index: MAX_NIC_INDEX, + }) + .context("invalid cvm.instance_id") +} + pub fn instance_id(configured: &str, run_path: &Path) -> String { if !configured.trim().is_empty() { return configured.trim().to_string(); From e0e305b54f240e503eecadb42cfc9ff7ad48d0a1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 02:35:13 -0700 Subject: [PATCH 11/34] docs(vmm): say what cvm.instance_id now decides It is what netd records on every host interface this VMM asks for, and what a collection uses to tell this instance's interfaces from another's. Two VMMs sharing one value on one host would each collect the other's running VMs. --- dstack/vmm/vmm.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index e89c30a6f..f0b7abd0f 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -46,7 +46,10 @@ max_allocable_memory_in_mb = 100_000 # MB qmp_socket = false # The user to run the VM as. If empty, the VM will be run as the current user. # Unique namespace when multiple dstack-vmm instances share one host. When -# empty, a stable value is derived from run_path. +# empty, a stable value is derived from run_path. It is what netd records on +# every host interface this VMM asks for, and what a collection uses to tell +# this instance's interfaces from another's -- so two VMMs sharing one value on +# one host would each collect the other's running VMs. May not contain ":". instance_id = "" # Network choices that deployment RPC callers may make. Macvtap is excluded # by default because libvirt nwfilter applies only to bridge interfaces. From f174c95d646723345e14f8e75e0a5ae823cb76ed Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 02:37:39 -0700 Subject: [PATCH 12/34] fix(cli): carry the new port mapping field through the client crates Additive on the wire and in the API: a client states a request, and `published` is the server's answer to it. --- dstack/crates/dstack-cli-core/src/ports.rs | 3 +++ dstack/crates/dstackup/src/install.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/dstack/crates/dstack-cli-core/src/ports.rs b/dstack/crates/dstack-cli-core/src/ports.rs index 142be1b16..03246c27b 100644 --- a/dstack/crates/dstack-cli-core/src/ports.rs +++ b/dstack/crates/dstack-cli-core/src/ports.rs @@ -77,6 +77,9 @@ pub fn parse_port(spec: &str) -> Result { host_port, vm_port, nic_index, + // A request, not an answer: the server reports what it actually + // published back through `GetInfo`. + published: None, }) } diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index b32f9fea0..41ee0f9d2 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -297,6 +297,9 @@ pub(crate) async fn cmd_install(mut o: InstallOpts, release_api_base_url: &str) // Unpinned: this deploys the node default topology, which // is one NIC, and the VMM resolves that itself. nic_index: None, + // A request, not an answer: the VMM reports what it + // actually published back through `GetInfo`. + published: None, }], ..Default::default() }; From a87404db95f85905659078e5dc708fe45a6c1725 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 02:55:42 -0700 Subject: [PATCH 13/34] fix(netd): stop a busy netd from reading as an absent one The accept loop served one connection at a time and `handle_request` blocks, so answering `hello` "before the lock" bought nothing: the connection was not *accepted* until whatever netd was doing finished. A collection may run for twenty seconds and a single `virsh` for thirty, while the capability probe gives up after five -- so any netd doing real work looked absent, and an absent netd is one whose teardown the VMM skips and whose deployments it refuses with "this netd does not forward host ports". Both wrong, and both introduced by the probe this branch added. One task per connection, with the blocking half on `spawn_blocking`. Serialization still holds, and now holds where it is actually stated: the operation lock is an flock, which contends between two open descriptions in one process exactly as it does between processes. What the single-connection loop added on top was head-of-line blocking and nothing else. The client side stops asking a question it does not need to ask. A release asks netd to release, rather than asking whether it may: the operation cannot be misread as absence, and only a refusal -- netd answering -- is worth a round trip, which is also exactly when the fallback matters. That removes the probe from every stop, and removes the case where the netd released one commit before this one, which does implement `remove_all`, was told it could not and given eight per-NIC removals instead. Reconciliation reads the same way. The one probe left in front of an action is the one that decides whether to act on the *absence* of an ownership record, which means nothing unless netd writes one -- so `attribution` is finally read where its documentation already claimed it was. Also: an orphaned nwfilter binding is now collected by default. It is the one unattributable thing that is unambiguously dead -- a record can only live on an interface, so it can never gain one, and nothing is using a binding with nothing to bind to. Left to the conservative default it was the single leak in this design that nothing could ever collect, since the VM it belonged to is gone and not even an operator could name it. And three narrower ones: a directory entry that cannot be read no longer silently drops a VM from the set that claims interfaces (the same care the `read_dir` error already got); a binding that answers on a different address than it was asked for is no longer reported as publishing the port, since an admin port on loopback and a published one differ only there; and two live VMMs configured with one `cvm.instance_id` now refuse to start rather than each collecting the other's running VMs. --- dstack/vmm/src/app.rs | 205 ++++++++++++++++++++++----------- dstack/vmm/src/app/vm_info.rs | 34 +++++- dstack/vmm/src/discovery.rs | 21 ++++ dstack/vmm/src/main.rs | 22 ++++ dstack/vmm/src/main_service.rs | 16 ++- dstack/vmm/src/netd.rs | 181 +++++++++++++++++++++++------ 6 files changed, 369 insertions(+), 110 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index b0700e828..bc8785dac 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -811,9 +811,12 @@ impl App { network.ingress = bound.clone(); for request in asked { if !bound.iter().any(|binding| { - binding.protocol == request.protocol - && binding.host_port == request.host_port - && binding.guest_port == request.guest_port + binding.answers( + &request.protocol, + &request.host_address, + request.host_port, + request.guest_port, + ) }) { warn!( vm_id = %vm.manifest.id, @@ -948,10 +951,40 @@ impl App { let mut ids: HashSet = self.lock().vms.keys().cloned().collect(); match fs::read_dir(self.vm_dir()) { Ok(entries) => { - for entry in entries.flatten() { - if entry.path().is_dir() { - if let Some(id) = entry.file_name().to_str() { - ids.insert(id.to_string()); + for entry in entries { + // Every error here is answered the same way the `read_dir` + // error below is, and for the same reason: an entry that + // cannot be read is a VM that might exist, and dropping it + // silently is how a collection deletes a running VM's + // networking over an unreadable directory entry. + let entry = match entry { + Ok(entry) => entry, + Err(error) => { + warn!("failed to read a VM directory entry: {error}; not collecting"); + return None; + } + }; + match entry.file_type() { + Ok(file_type) if !file_type.is_dir() => continue, + Ok(_) => {} + Err(error) => { + warn!( + name = ?entry.file_name(), + "failed to stat a VM directory entry: {error}; not collecting" + ); + return None; + } + } + match entry.file_name().into_string() { + Ok(id) => { + ids.insert(id); + } + // A VM ID is ASCII, so a name that is not UTF-8 is not + // a VM directory -- but it is also not something to + // decide a deletion around. + Err(name) => { + warn!(?name, "unreadable VM directory name; not collecting"); + return None; } } } @@ -986,24 +1019,27 @@ impl App { /// from overlapping a prepare; the ordering here is what keeps it from /// racing the decision. pub(crate) async fn reconcile_netd_interfaces(&self) { - let reachability = self.netd_capabilities().await; - if !reachability.is_reachable() { - debug!("no netd to reconcile interfaces with"); - return; - } - if !reachability.supports("gc") { - warn!( - netd = %reachability.describe(), - "netd on this node cannot collect interfaces no VM claims; a VM removed while \ - this VMM was down leaves its host interfaces behind until it runs a newer netd" - ); - return; - } let Some(live) = self.claimable_vm_ids() else { return; }; let policy = if self.config.netd.collect_unattributed { - netd::UnattributedPolicy::Remove + // The one decision worth a round trip before making it. Acting on + // the *absence* of an ownership record only means anything if the + // netd that built these interfaces writes one; asked of a netd that + // does not, "no record" describes every interface on the host -- + // including another instance's running VMs. + let reachability = self.netd_capabilities().await; + if reachability.records_ownership() { + netd::UnattributedPolicy::Remove + } else { + warn!( + netd = %reachability.describe(), + "netd.collect_unattributed is set, but this netd does not record which VM an \ + interface belongs to, so an interface with no record says nothing about what \ + is using it; collecting only what is attributed" + ); + netd::UnattributedPolicy::Keep + } } else { netd::UnattributedPolicy::Keep }; @@ -1037,7 +1073,19 @@ impl App { Err(error) if netd::is_unreachable(&error) => { debug!("no netd to reconcile interfaces with") } - Err(error) => warn!("failed to reconcile netd-managed interfaces: {error:#}"), + Err(error) => { + // Same reading a release gets: it answered, so now the question + // is whether the answer means it cannot do this at all. + if self.netd_capabilities().await.supports("gc") { + warn!("failed to reconcile netd-managed interfaces: {error:#}"); + } else { + warn!( + "netd on this node cannot collect interfaces no VM claims; a VM removed \ + while this VMM was down leaves its host interfaces behind until the node \ + runs a newer netd" + ); + } + } } } @@ -1056,34 +1104,40 @@ impl App { /// reach. `recorded` is not the source of truth -- it is what a netd too /// old to sweep by identity has to be told instead. pub(crate) async fn release_vm_interfaces(&self, vm_id: &str, recorded: &[Networking]) { - let reachability = self.netd_capabilities().await; - if !reachability.is_reachable() { - debug!(vm_id, "no netd to release interfaces from"); - return; - } - if reachability.supports("remove_all") { - match netd::remove_all( - &self.config.netd.socket, - &self.config.cvm.instance_id, - vm_id, - ) - .await - { - Ok(0) => {} - Ok(removed) => info!(vm_id, removed, "released netd-managed interfaces"), - Err(error) if netd::is_unreachable(&error) => { - debug!(vm_id, %error, "no netd to release interfaces from") - } - Err(error) => { + // Ask for the release, rather than asking whether it can be asked for. + // A probe first would put a second round trip in front of every stop + // and -- worse -- would make a *busy* netd look like an absent one and + // skip the release entirely. The operation itself cannot be misread + // that way: it succeeds, or it says netd is not there, or netd answers + // with a refusal, and only the last of those has a fallback. + match netd::remove_all( + &self.config.netd.socket, + &self.config.cvm.instance_id, + vm_id, + ) + .await + { + Ok(0) => {} + Ok(removed) => info!(vm_id, removed, "released netd-managed interfaces"), + Err(error) if netd::is_unreachable(&error) => { + debug!(vm_id, %error, "no netd to release interfaces from") + } + Err(error) => { + // It answered, so it is up. Now the question is worth a round + // trip: an operation a netd does not have looks exactly like + // one that failed, and only one of those has a fallback. A netd + // that has this operation and failed at it is a failure to + // report, not a reason to send it eight more requests. + if self.netd_capabilities().await.supports("remove_all") { warn!( vm_id, "failed to release netd-managed interfaces: {error:#}" - ) + ); + } else { + self.release_recorded_interfaces(vm_id, recorded).await; } } - return; } - self.release_recorded_interfaces(vm_id, recorded).await; } /// The teardown a netd that cannot sweep by identity gets. @@ -2435,10 +2489,13 @@ mod tests { app.release_vm_interfaces("vm-1", &[]).await; let operations = netd.operations(); - assert_eq!(operations[0], "hello", "capability is asked, not inferred"); - assert!( - !operations.contains(&"remove_all".to_string()), - "an operation it does not have is not sent" + assert_eq!( + operations[0], "remove_all", + "the release is asked for, not asked about" + ); + assert_eq!( + operations[1], "hello", + "only a refusal is worth a question, and then it is asked rather than inferred" ); assert_eq!( operations.iter().filter(|op| *op == "remove").count(), @@ -2447,8 +2504,12 @@ mod tests { ); } + /// A probe in front of every stop is a second round trip on the hot path, + /// and -- because netd answers one caller at a time -- a *busy* netd would + /// answer it too slowly and be taken for an absent one, skipping the + /// release entirely. The operation cannot be misread that way. #[tokio::test] - async fn a_netd_that_sweeps_is_asked_once_and_by_identity() { + async fn a_netd_that_sweeps_is_asked_to_without_a_question_first() { let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ "hello", "remove", @@ -2457,8 +2518,8 @@ mod tests { let app = app_talking_to(netd.socket()); app.release_vm_interfaces("vm-1", &[]).await; - assert_eq!(netd.operations(), vec!["hello", "remove_all"]); - let sweep = &netd.seen()[1]; + assert_eq!(netd.operations(), vec!["remove_all"]); + let sweep = &netd.seen()[0]; assert_eq!(sweep["vm_id"], "vm-1"); assert_eq!(sweep["instance_id"], "test-instance"); // A sweep names no NIC: reaching the indices the caller can no longer @@ -2466,20 +2527,17 @@ mod tests { assert!(sweep.get("nic_index").is_none()); } - /// One probe covers a window rather than one call, or netd's serialized - /// accept loop services a connection per VM per operation. + /// Where a probe is unavoidable it covers a window rather than one call, + /// or netd's accept loop services a connection per VM per operation. #[tokio::test] async fn the_capability_answer_is_reused() { - let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ - "hello", - "remove_all", - ])); + let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); let app = app_talking_to(netd.socket()); app.release_vm_interfaces("vm-1", &[]).await; app.release_vm_interfaces("vm-2", &[]).await; assert_eq!( - netd.operations(), - vec!["hello", "remove_all", "remove_all"], + netd.operations().iter().filter(|op| *op == "hello").count(), + 1, "asked once, acted on twice" ); } @@ -2495,18 +2553,16 @@ mod tests { ])); let app = app_talking_to(old.socket()); app.reconcile_netd_interfaces().await; - assert_eq!( - old.operations(), - vec!["hello"], - "an operation it does not have is not sent" - ); + // Asked for, refused, and only then asked about -- so that "cannot do + // this" is reported as itself rather than as a collection that failed. + assert_eq!(old.operations(), vec!["gc", "hello"]); let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&["hello", "gc"])); let app = app_talking_to(netd.socket()); app.reconcile_netd_interfaces().await; - let request = &netd.seen()[1]; - assert_eq!(request["operation"], "gc"); + assert_eq!(netd.operations(), vec!["gc"]); + let request = &netd.seen()[0]; assert_eq!(request["instance_id"], "test-instance"); assert_eq!(request["live_vm_ids"].as_array().unwrap().len(), 0); assert_eq!(request["dry_run"], false); @@ -2519,12 +2575,23 @@ mod tests { async fn collecting_unattributed_interfaces_is_the_operators_to_ask_for() { let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&["hello", "gc"])); - let mut app = app_talking_to(netd.socket()); - let mut config = (*app.config).clone(); + let mut config = test_config(netd.socket(), netd.socket().parent().unwrap()); config.netd.collect_unattributed = true; - app = App::new(config, SupervisorClient::new("http://127.0.0.1:0")); + let app = App::new(config.clone(), SupervisorClient::new("http://127.0.0.1:0")); app.reconcile_netd_interfaces().await; + assert_eq!(netd.operations(), vec!["hello", "gc"]); assert_eq!(netd.seen()[1]["unattributed"], "remove"); + + // Acting on the absence of a record only means anything if the netd + // that built these interfaces writes one. Asked of a netd that does + // not, "no record" describes every interface on the host -- including + // another instance's running VMs. + let anonymous = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::anonymous(&["hello", "gc"])); + config.netd.socket = anonymous.socket().to_path_buf(); + let app = App::new(config, SupervisorClient::new("http://127.0.0.1:0")); + app.reconcile_netd_interfaces().await; + assert_eq!(anonymous.seen()[1]["unattributed"], "keep"); } /// The window a launch spends between reading "not running" and actually diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 4d955bb64..f41e4ce8c 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -144,10 +144,14 @@ fn published_at(mapping: &crate::app::PortMapping, networks: &[Networking]) -> b // QEMU carries these itself, for as long as it is up. return true; } + let host_address = mapping.address.to_string(); network.ingress.iter().any(|binding| { - binding.protocol == mapping.protocol.as_str() - && binding.host_port == mapping.from - && binding.guest_port == mapping.to + binding.answers( + mapping.protocol.as_str(), + &host_address, + mapping.from, + mapping.to, + ) }) } @@ -411,6 +415,30 @@ mod tests { // A mapping naming a NIC the VM no longer has is answered by nobody. assert!(!published_at(&mapping(443, Some(7)), &networks)); + + // An admin port on loopback and a published one differ only in the + // address, so a netd that narrowed 0.0.0.0 to 127.0.0.1 has not met + // the request -- and reporting it as met would report a port as + // reachable from the network when it is not. + let mut narrowed = nic(NetworkingMode::Bridge); + narrowed.ingress = vec![IngressBinding { + protocol: "tcp".into(), + host_address: "127.0.0.1".into(), + host_port: 443, + guest_port: 8080, + }]; + assert!(!published_at(&mapping(443, None), &[narrowed])); + + // An address netd did not state at all is a netd that echoes less than + // it was told, not one that narrowed anything. + let mut silent = nic(NetworkingMode::Bridge); + silent.ingress = vec![IngressBinding { + protocol: "tcp".into(), + host_address: String::new(), + host_port: 443, + guest_port: 8080, + }]; + assert!(published_at(&mapping(443, None), &[silent])); } /// Custom mode hands the operator the whole netdev string and the VMM never diff --git a/dstack/vmm/src/discovery.rs b/dstack/vmm/src/discovery.rs index 181e7da52..f17610c18 100644 --- a/dstack/vmm/src/discovery.rs +++ b/dstack/vmm/src/discovery.rs @@ -42,6 +42,11 @@ pub struct VmmInstanceInfo { pub run_path: String, /// Node name from configuration. pub node_name: String, + /// The namespace this VMM's host interfaces are recorded under. Two live + /// instances must not share one: it is the name space their TAP names are + /// derived in and the only thing a collection can tell them apart by. + #[serde(default)] + pub instance_id: String, /// VMM version string. pub version: String, /// Unix timestamp (seconds) when the instance started. @@ -64,6 +69,7 @@ impl DiscoveryRegistration { run_path: &Path, node_name: &str, version: &str, + instance_id: &str, ) -> Result { let dir = discovery_dir(); fs_err::create_dir_all(&dir).context("failed to create discovery directory")?; @@ -82,6 +88,7 @@ impl DiscoveryRegistration { image_path: image_path.to_string_lossy().to_string(), run_path: run_path.to_string_lossy().to_string(), node_name: node_name.to_string(), + instance_id: instance_id.to_string(), version: version.to_string(), started_at: std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -110,6 +117,20 @@ impl Drop for DiscoveryRegistration { } } +/// Every VMM instance currently registered as alive on this host. +pub fn live_instances() -> Vec { + let Ok(entries) = fs::read_dir(discovery_dir()) else { + return Vec::new(); + }; + entries + .flatten() + .filter(|entry| entry.path().extension().and_then(|e| e.to_str()) == Some("json")) + .filter_map(|entry| fs::read_to_string(entry.path()).ok()) + .filter_map(|content| serde_json::from_str::(&content).ok()) + .filter(|info| Path::new(&format!("/proc/{}", info.pid)).exists()) + .collect() +} + /// Clean up stale discovery files from dead processes. pub fn cleanup_stale_registrations() { let dir = discovery_dir(); diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index b926ba5c7..8cdba5c45 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -353,6 +353,27 @@ async fn main() -> Result<()> { // Preserve the existing startup validation. The broader static checks are // opt-in through `check-config` until they have seen wider deployment use. netd::validate_instance_id(&config.cvm.instance_id)?; + // Two live VMMs sharing one instance ID share the name space their host + // interfaces are derived in: each would build TAPs at names the other can + // also produce, and each collection would delete the other's running VMs + // because the ownership record -- the only thing that can tell two + // instances apart -- would name the collector. Derived from `run_path` + // this cannot happen; it takes a copied `vmm.toml` that states one. + for peer in discovery::live_instances() { + if peer.instance_id == config.cvm.instance_id + && peer.run_path != config.run_path.to_string_lossy() + { + anyhow::bail!( + "cvm.instance_id '{}' is already in use by the VMM running at {} (pid {}). It is \ + the name space this VMM's host interfaces are derived in, so sharing one would \ + have each instance delete the other's running VMs' networking. Leave it empty to \ + derive it from run_path", + config.cvm.instance_id, + peer.run_path, + peer.pid + ); + } + } config .host_api .validate() @@ -420,6 +441,7 @@ async fn main() -> Result<()> { &config.run_path, &config.node_name, &app_version(), + &config.cvm.instance_id, ) { Ok(registration) => Some(registration), Err(err) => { diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 85866bc19..9d7f76e12 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -874,11 +874,19 @@ impl RpcHandler { if reachability.forwards_ingress() { return Ok(()); } + let named = needs_netd.join(", "); + // "It does not forward" and "it could not be asked" call for different + // fixes, and only one of them is about the deployment. + if !reachability.is_reachable() { + bail!( + "port mapping {named} enters through a bridge NIC, which only netd can publish, \ + and netd could not be reached to ask whether it does" + ); + } bail!( - "port mapping {} enters through a bridge NIC, which only netd can publish, and the \ - netd on this node does not forward host ports ({}). Put the mapping on a user-mode \ - NIC with @, or deploy a netd that forwards", - needs_netd.join(", "), + "port mapping {named} enters through a bridge NIC, which only netd can publish, and \ + the netd on this node does not forward host ports ({}). Put the mapping on a \ + user-mode NIC with @, or run a netd that forwards", reachability.describe() ) } diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 7b8f13316..9a7fc6c19 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -71,14 +71,18 @@ const TAP_DIGEST_CHARS: usize = 12; const ALIAS_PREFIX: &str = "dstack1"; /// What the kernel stores in an interface alias, minus the terminator. const MAX_IFALIAS: usize = 255; -/// How long one sweep or collection may run before it reports what it has done -/// and stops. +/// How long a sweep or collection keeps starting work on new interfaces. /// -/// Under the operation lock and inside a serialized accept loop, so an -/// unbounded pass is not slow, it is an outage: one hung `virsh` per interface -/// would hold every other VM's prepare and remove behind it, and the caller -/// that asked has long since timed out. Partial progress reported honestly -/// beats total progress nobody is still waiting for. +/// A bound on when the pass stops, not on how long one interface takes: an +/// interface it has already begun is still bounded only by `COMMAND_TIMEOUT` +/// per helper invocation, so a pass that passes this check at 19.9s can run on +/// for as long as the `ip` and `virsh` calls it has started take to time out. +/// +/// What it protects is the caller's patience against the operation lock. Every +/// other prepare and remove on the host waits behind a pass, and the caller +/// that asked for this one gave up at thirty seconds; work done after that is +/// work nobody is waiting for, done while everybody waits. Partial progress +/// reported honestly beats total progress reported to nobody. const COLLECTION_DEADLINE: Duration = Duration::from_secs(20); #[derive(Debug, Clone, Serialize, Deserialize)] @@ -247,7 +251,9 @@ pub struct Capabilities { pub ingress: bool, /// Whether it records ownership on the interface itself. A whole-host /// collection cannot tell one VMM instance's interfaces from another's - /// without it, so a netd that says no is never asked to collect. + /// without it, so a netd that says no is never asked to act on the + /// *absence* of a record: "no record" would describe every interface on + /// the host, including another instance's running VMs. #[serde(default)] pub attribution: bool, } @@ -258,6 +264,29 @@ impl Capabilities { } } +impl IngressBinding { + /// Whether this binding answers a request for one host port. + /// + /// The address is part of the answer, not decoration: an admin port bound + /// to loopback and a published one differ only there, so a netd that + /// narrowed `0.0.0.0` to `127.0.0.1` has not met the request, and + /// reporting it as met would report a port as reachable from the network + /// when it is not. An address netd did not state at all is not a narrowing; + /// it is a netd that echoes less than it was told. + pub fn answers( + &self, + protocol: &str, + host_address: &str, + host_port: u16, + guest_port: u16, + ) -> bool { + self.protocol == protocol + && self.host_port == host_port + && self.guest_port == guest_port + && (self.host_address.is_empty() || self.host_address == host_address) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrepareMacvtapRequest { #[serde(flatten)] @@ -709,6 +738,12 @@ impl Reachability { /// Whether asking this netd to forward host ports is meaningful. Unknown /// counts as no: a caller that assumed yes would report ports as published /// on the strength of never having asked. + /// Whether it records which VM an interface belongs to. Unknown counts as + /// no, for the same reason `forwards_ingress` does. + pub fn records_ownership(&self) -> bool { + matches!(self, Self::Capable(capabilities) if capabilities.attribution) + } + pub fn forwards_ingress(&self) -> bool { matches!(self, Self::Capable(capabilities) if capabilities.ingress) } @@ -821,16 +856,33 @@ pub async fn serve(config: NetdConfig) -> Result<()> { None => bind_listener(&config)?, }; info!(address = ?listener.local_addr()?, "netd listening"); + let config = std::sync::Arc::new(config); loop { let (mut stream, _) = listener.accept().await?; - // This timeout bounds async socket reads and writes. handle_request is - // synchronous, so helper execution is bounded separately by - // COMMAND_TIMEOUT rather than preempted by this future timeout. - match timeout(CONNECTION_TIMEOUT, serve_connection(&config, &mut stream)).await { - Ok(Ok(())) => {} - Ok(Err(error)) => warn!(%error, "netd connection failed"), - Err(_) => warn!("netd connection timed out"), - } + // One task per connection, so a request that takes minutes does not + // stop the next one from being *accepted*. + // + // Serialization still holds where it matters, and holds where it is + // actually stated: `handle_request` takes the operation lock, which is + // an flock and blocks between two open descriptions in one process + // just as it does between processes. What a single-connection accept + // loop added on top of that was head-of-line blocking -- a caller + // asking netd a question it answers in microseconds waited for whatever + // netd happened to be doing, and gave up believing netd was not there. + // A collection can run for twenty seconds and a `virsh` for thirty, so + // this was not a corner: it made a busy netd indistinguishable from an + // absent one, and teardown skips an absent netd. + let config = config.clone(); + tokio::spawn(async move { + // Bounds the socket reads and writes. It cannot preempt + // `handle_request`, which is synchronous and bounded separately by + // COMMAND_TIMEOUT per helper invocation. + match timeout(CONNECTION_TIMEOUT, serve_connection(&config, &mut stream)).await { + Ok(Ok(())) => {} + Ok(Err(error)) => warn!(%error, "netd connection failed"), + Err(_) => warn!("netd connection timed out"), + } + }); } } @@ -865,7 +917,10 @@ fn bind_listener(config: &NetdConfig) -> Result { Ok(listener) } -async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Result<()> { +async fn serve_connection( + config: &std::sync::Arc, + stream: &mut UnixStream, +) -> Result<()> { // Access is authorized by the Unix socket's owner, group, and mode. Any // process that can connect is trusted with the complete netd protocol. let outcome = match read_request(stream).await { @@ -877,7 +932,18 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul debug!("netd liveness probe"); return Ok(()); } - Ok(Some(request)) => handle_request(config, request), + Ok(Some(request)) => { + // `handle_request` shells out to `ip` and `virsh` and waits on the + // operation lock, so it can block for as long as those take. On a + // runtime worker that would stall every other connection's reads + // and writes, which is the head-of-line blocking this daemon just + // stopped having. + let config = config.clone(); + match tokio::task::spawn_blocking(move || handle_request(&config, request)).await { + Ok(outcome) => outcome, + Err(error) => Err(anyhow::anyhow!("netd worker failed: {error}")), + } + } // A request that arrived but could not be understood still gets an // answer. A VMM newer than this netd sends operations it does not // know, and "unknown variant `prepare_foo`" is what tells the operator @@ -940,7 +1006,7 @@ fn capabilities() -> Capabilities { fn handle_request(config: &NetdConfig, request: Request) -> Result { // Answered before the lock. A caller asks this to find out whether netd - // can do the thing it is about to ask for, and making that wait behind a + // can do the thing it is about to ask for, and making it wait behind a // running collection would put a whole-host sweep in front of every // launch's first question. if matches!(request, Request::Hello) { @@ -1308,7 +1374,16 @@ fn gc_plan( // ownership was recorded survives the upgrade that introduced it. _ => { let names = derived.get_or_insert_with(live_names); - names.contains(&record.tap) || policy == UnattributedPolicy::Keep + names.contains(&record.tap) + // An nwfilter binding whose interface is gone is the one + // unattributable thing that is also unambiguously dead: a + // record can only ever live on an interface, so this can + // never gain one, and nothing is using a binding with + // nothing to bind to. Left to the conservative default it + // would be the one leak in this design that nothing could + // ever collect -- the VM it belonged to is gone, so not + // even an operator could name it. + || (record.kind != "binding" && policy == UnattributedPolicy::Keep) } }; if !keep { @@ -1406,12 +1481,18 @@ fn collect_garbage( Err(error) => warn!(tap = %record.tap, "failed to collect: {error:#}"), } } - if unattributed > 0 && policy == UnattributedPolicy::Keep { + let left_alone = unattributed.saturating_sub( + taken + .iter() + .filter(|record| record.instance_id.is_none()) + .count(), + ); + if left_alone > 0 && policy == UnattributedPolicy::Keep { info!( %instance_id, - unattributed, - "interfaces carry no ownership record and were left alone; they gain one when their \ - VM next launches" + unattributed = left_alone, + "interfaces carry no ownership record and were left alone; each gains one when its VM \ + next launches" ); } Ok(Outcome::Collected { @@ -1861,6 +1942,8 @@ pub(crate) mod testing { Capable { operations: Vec, ingress: bool, + /// Whether it claims to record who an interface belongs to. + attribution: bool, }, /// Reached, but predates `hello`: every unknown operation is an error, /// exactly as `serde` produces one. @@ -1872,6 +1955,7 @@ pub(crate) mod testing { Self::Capable { operations: operations.iter().map(|name| name.to_string()).collect(), ingress: false, + attribution: true, } } @@ -1880,6 +1964,24 @@ pub(crate) mod testing { Self::Capable { operations, .. } => Self::Capable { operations, ingress: true, + attribution: true, + }, + other => other, + } + } + + /// A netd that builds interfaces without recording whose they are -- + /// a third-party one, or this one before it did. + pub(crate) fn anonymous(operations: &[&str]) -> Self { + match Self::capable(operations) { + Self::Capable { + operations, + ingress, + .. + } => Self::Capable { + operations, + ingress, + attribution: false, }, other => other, } @@ -1949,7 +2051,7 @@ pub(crate) mod testing { fn answer(behavior: &Behavior, request: &Value) -> Value { let operation = request["operation"].as_str().unwrap_or_default(); - let (operations, ingress) = match behavior { + let (operations, ingress, attribution) = match behavior { Behavior::Legacy => { return match operation { // What the real thing answers for an operation it knows. @@ -1965,7 +2067,8 @@ pub(crate) mod testing { Behavior::Capable { operations, ingress, - } => (operations, *ingress), + attribution, + } => (operations, *ingress, *attribution), }; if !operations.iter().any(|name| name == operation) { return json!({ @@ -1980,7 +2083,7 @@ pub(crate) mod testing { "version": "fake-netd", "operations": operations, "ingress": ingress, - "attribution": true, + "attribution": attribution, }, }), "prepare_bridge" | "prepare_macvtap" => { @@ -2228,7 +2331,7 @@ mod tests { drop(client); let result = timeout( Duration::from_secs(1), - serve_connection(&NetdConfig::default(), &mut server), + serve_connection(&std::sync::Arc::new(NetdConfig::default()), &mut server), ) .await; assert!(result.is_ok(), "disconnected peer blocked the handler"); @@ -2251,7 +2354,7 @@ mod tests { .await .unwrap(); client.shutdown().await.unwrap(); - serve_connection(&NetdConfig::default(), &mut server) + serve_connection(&std::sync::Arc::new(NetdConfig::default()), &mut server) .await .unwrap(); @@ -2677,19 +2780,29 @@ mod tests { } /// A binding outlives the interface it was bound to, so it is the one piece - /// of state that can never carry a record, and the one a collection over - /// interfaces alone would always miss. + /// of state that can never carry a record -- and therefore the one thing a + /// conservative default would strand forever. It is also the one + /// unattributable thing that is unambiguously dead: nothing is using a + /// binding with nothing to bind to, and the VM it belonged to is gone, so + /// not even an operator could name it to remove it by hand. #[test] - fn an_orphaned_binding_is_collectable_but_not_by_default() { + fn an_orphaned_binding_is_collected_even_by_default() { let mut binding = record("dt00000000beef", None, None); binding.kind = "binding".to_string(); binding.bound = true; let records = [binding]; - assert!(plan(&records, &[], UnattributedPolicy::Keep).is_empty()); assert_eq!( - plan(&records, &[], UnattributedPolicy::Remove), + plan(&records, &[], UnattributedPolicy::Keep), vec!["dt00000000beef".to_string()] ); + + // Except where the name is one a live VM would occupy: a filtered + // interface is rebuilt binding-first, so the binding can legitimately + // exist for a moment without one. + let live_tap = tap_name(&identity("ours", "live-vm", 0)); + let mut binding = record(&live_tap, None, None); + binding.kind = "binding".to_string(); + assert!(plan(&[binding], &["live-vm"], UnattributedPolicy::Keep).is_empty()); } /// Everything else here reasons about strings. This puts the reasoning From 3f63b2685e2fda53047ebae6e94e3f4eebc66bc3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 03:03:27 -0700 Subject: [PATCH 14/34] fix(netd): stop the connection timeout from throwing away work it cannot cancel It wrapped the whole exchange, which reads as a bound on the request and is not one: `handle_request` is synchronous and shells out, so the timeout could not cancel it. All it did was drop the connection at thirty-five seconds while the work went on running to completion -- and then log "netd connection timed out" for a request netd in fact finished. A caller has its own deadline and has gone by then; what this cost was netd's own account of what it did. Now the timeouts bound the socket reads and writes, which is what the comment said they were for. Observed against a netd wedged in a twenty-five second helper: `hello` answers in 0.00s, a queued `list` answers at 49s rather than being cut off at 35s, and the log says what happened. --- dstack/vmm/src/netd.rs | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 9a7fc6c19..c6c7210ca 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -874,13 +874,8 @@ pub async fn serve(config: NetdConfig) -> Result<()> { // absent one, and teardown skips an absent netd. let config = config.clone(); tokio::spawn(async move { - // Bounds the socket reads and writes. It cannot preempt - // `handle_request`, which is synchronous and bounded separately by - // COMMAND_TIMEOUT per helper invocation. - match timeout(CONNECTION_TIMEOUT, serve_connection(&config, &mut stream)).await { - Ok(Ok(())) => {} - Ok(Err(error)) => warn!(%error, "netd connection failed"), - Err(_) => warn!("netd connection timed out"), + if let Err(error) = serve_connection(&config, &mut stream).await { + warn!(%error, "netd connection failed"); } }); } @@ -917,13 +912,26 @@ fn bind_listener(config: &NetdConfig) -> Result { Ok(listener) } +/// Serves one connection. +/// +/// The timeouts bound the socket reads and writes and nothing else. They used +/// to wrap the whole exchange, which read as a bound on the request but was +/// not one: `handle_request` is synchronous and shells out, so the timeout +/// could not cancel it -- it only threw away the answer to work that went on +/// running. A caller has its own deadline and will have gone by then; what it +/// cost was netd's own log, which reported "timed out" for a request it in +/// fact completed. Helper execution is bounded where it can be, by +/// COMMAND_TIMEOUT per invocation. async fn serve_connection( config: &std::sync::Arc, stream: &mut UnixStream, ) -> Result<()> { // Access is authorized by the Unix socket's owner, group, and mode. Any // process that can connect is trusted with the complete netd protocol. - let outcome = match read_request(stream).await { + let request = timeout(CONNECTION_TIMEOUT, read_request(stream)) + .await + .context("timed out reading a netd request")?; + let outcome = match request { // A peer that connects and closes without sending is the VMM's // reachability check: netd that died leaves its socket behind, so the // VMM connects to tell the two apart. Answering that with a parse error @@ -969,8 +977,12 @@ async fn serve_connection( } }; let encoded = serde_json::to_vec(&response)?; - stream.write_all(&encoded).await?; - stream.shutdown().await?; + timeout(CONNECTION_TIMEOUT, async { + stream.write_all(&encoded).await?; + stream.shutdown().await + }) + .await + .context("timed out answering a netd request")??; Ok(()) } From 17e8692f78feeebb6c1ad204f3718373af784a53 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 03:05:22 -0700 Subject: [PATCH 15/34] docs(vmm): note that an orphaned nwfilter binding is always collected --- docs/bridge-networking.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 06a79abb3..b7ccec05f 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -284,6 +284,7 @@ Three rules: | Another VMM instance's | Never touched. Several VMM instances share one `netd`, and the record is the only thing that can tell that instance's *running* VM from garbage | | This instance's, for a VM it no longer has | Collected | | Nothing that checks out | Kept and reported. Set `netd.collect_unattributed = true` to collect it | +| An nwfilter binding whose interface is gone | Always collected. A record can only live on an interface, so this can never gain one — and nothing is using a binding with nothing to bind to | An interface with no record is not nobody's: before `netd` recorded ownership every interface looked like this, and on a host with two VMM instances one of From 267c17369f852eb7b24a326ba1a6639f00348dce Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 03:39:23 -0700 Subject: [PATCH 16/34] fix(netd): make the binding listing work, and decide collections where the lock is Two findings from review, and the second reshapes the design. `virsh nwfilter-binding-list` accepts no options at all -- `--name` is not one of them, and asking for it fails the whole call. So `existing_bindings` has always returned `None`, which was survivable until this branch made a sweep read that as "libvirt is down, skip the bindings": every stop, every pre-launch release and every removal then deleted the TAP and left the binding, permanently for a VM that was removed. Verified against virsh 10.0.0 on a node holding seven real bindings. It parses the table now, narrowed to netd's own name space so a header, a rule line or a moved column cannot produce a name; and a listing that could not be produced no longer decides whether deletions are attempted. The collection moves out of netd. A collection decided inside netd is decided against a set of live VMs that was true when the caller *sent* it -- netd runs the request when it wins the operation lock, which may be much later -- so a VM created in between is absent from the set and present on the host, and its TAP is deleted while QEMU is starting on it. That was safe at startup, where the API is not yet served, and the interval pass ran the same code with the API served. netd cannot close it: the lock that closes it is the VMM's per-VM launch lock, and netd has no way to take one. So the VMM asks `list` and decides for itself, per VM, under exactly that lock, re-reading whether it claims the VM while holding it. A launch and a collection of the same VM can no longer both believe they are alone. It also deletes `Gc`, `gc_plan`, `live_interface_names`, `collect_garbage`, `UnattributedPolicy`, `Collection`, dry runs, and the `collect_unattributed` knob: the collection is `list` plus the whole-VM sweep a stop already uses. What is left over is what nothing can attribute, and nothing decides about it -- correct, and previously a dead end, since `netd list` shows no VM to name in `remove-vm`. `netd remove-interface ` is the operator's way to say what a machine cannot work out. Also: a VM directory that is not there is no longer read as "no VMs". "Never ran" and "the volume is not mounted yet" are the same error, and only one of them means there is nothing to collect. --- docs/bridge-networking.md | 40 +-- docs/libvirt-network-filter.md | 8 +- dstack/vmm/src/app.rs | 321 +++++++++++++-------- dstack/vmm/src/config.rs | 10 - dstack/vmm/src/main.rs | 18 ++ dstack/vmm/src/netd.rs | 493 ++++++++------------------------- dstack/vmm/vmm.toml | 7 +- 7 files changed, 357 insertions(+), 540 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index b7ccec05f..c685cefd7 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -271,30 +271,38 @@ sudo dstack-vmm netd remove-vm --instance path-3f9a1c8e7d2b4a60 --vm 0a1b2c3d4e5 ### Collection -The VMM asks `netd` to collect interfaces no VM of its claims: once at startup, -after it has loaded its VMs and before it serves its API, and then every -`netd.reconcile_interval_secs`. That is what reaches an interface no per-VM -teardown can — one whose VM was removed while the VMM was down, or whose -workdir was deleted by hand. - -Three rules: +The VMM reconciles what `netd` holds against the VMs it has: once at startup, +after loading them, and then every `netd.reconcile_interval_secs`. That is what +reaches an interface no per-VM teardown can — one whose VM was removed while +the VMM was down, or whose workdir was deleted by hand. + +It asks `netd list` and decides for itself, rather than asking `netd` to decide. +The decision is only safe under the VMM's per-VM launch lock, which `netd` has +no way to take: a collection decided inside `netd` would be decided against a +set of live VMs that was true when the request was *sent*, and `netd` runs it +when it wins the operation lock — possibly much later, by which time a VM +created in between is absent from the set and present on the host. Here each +VM is re-checked while holding the lock its own launch holds, so a launch and a +collection of the same VM cannot both believe they are alone. | What the interface is recorded as | What happens | | --- | --- | | Another VMM instance's | Never touched. Several VMM instances share one `netd`, and the record is the only thing that can tell that instance's *running* VM from garbage | -| This instance's, for a VM it no longer has | Collected | -| Nothing that checks out | Kept and reported. Set `netd.collect_unattributed = true` to collect it | -| An nwfilter binding whose interface is gone | Always collected. A record can only live on an interface, so this can never gain one — and nothing is using a binding with nothing to bind to | +| This instance's, for a VM it no longer has | Collected, by the same whole-VM sweep a stop uses | +| Nothing that checks out | Left alone. No VMM can tell whose it is, so no VMM decides about it | An interface with no record is not nobody's: before `netd` recorded ownership every interface looked like this, and on a host with two VMM instances one of -them may be the other's running VM. Turn `collect_unattributed` on only where a -single VMM instance owns the host. +them may be the other's running VM. They are reported at each pass and listed +by `netd list` with `-` for instance and VM; an operator who can tell what one +is removes it by name: + +```bash +sudo dstack-vmm netd remove-interface dtc41d9e0b7a52 +``` -Upgrading is safe without touching anything: a collection also derives the -names its own live VMs would occupy and keeps those, so a fleet running from -before this existed survives the first pass, and each interface gains a record -the next time its VM launches. +Nothing accumulates: each interface gains a record the next time its VM +launches, so the set only shrinks. ### Mixing networking modes diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index bf99a3ab0..7bad88ba7 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -20,9 +20,11 @@ The measurable acceptance criteria are: that TAP carries a binding. - `network_filter = "libvirt"` creates the TAP and filter binding before QEMU is submitted to Supervisor, and uses QEMU `-netdev tap`. -- An nwfilter binding outlives the TAP it was bound to, so teardown and - collection both clear a binding whose interface is already gone. `dstack-vmm - netd list` shows one as a `binding` row with no interface. +- An nwfilter binding outlives the TAP it was bound to, so a teardown clears + the binding at every name that VM could have used, whether or not the + interface is still there. `dstack-vmm netd list` shows a binding whose + interface is already gone as a `binding` row; remove one with + `dstack-vmm netd remove-interface `. - A failed TAP or filter setup prevents QEMU from starting and rolls back all interfaces prepared for that VM. - Normal stop and removal delete the filter binding and TAP. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index bc8785dac..a4282da9a 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -713,9 +713,11 @@ impl App { let response = match netd::request(&self.config.netd.socket, &request).await { Ok(response) => response, Err(error) => { - // The client may have timed out while netd was still finishing - // this Prepare. Remove the in-flight identity first; netd's - // serialized accept loop processes it after Prepare completes. + // The client may have timed out while netd was still + // finishing this Prepare. Remove the in-flight identity + // first: the operation lock makes netd run that removal + // after the Prepare it is undoing, whatever order the two + // connections arrived in. if let Err(cleanup_error) = netd::request( &self.config.netd.socket, &NetdRequest::Remove { @@ -913,8 +915,8 @@ impl App { /// What this node's netd can do, cached for [`NETD_PROBE_TTL`]. /// /// Asked rather than inferred, and asked once per window rather than per - /// VM: every launch, teardown and reconciliation needs the answer, and the - /// probe is a connection netd's serialized accept loop has to service. + /// VM: it is a round trip, and the paths that want it are the ones already + /// making one. /// An unreachable answer is not cached -- a failed connect costs nothing, /// and holding on to it would keep a VMM blind to the netd an operator /// just started. @@ -989,9 +991,15 @@ impl App { } } } - // A VMM that has never run has no directory and no VMs, which is a - // knowable answer rather than an unknowable one. - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + // "Never ran" and "the volume is not mounted yet" produce the same + // error, and only one of them means there are no VMs. A VMM that + // has never run has nothing to collect either way, so declining + // costs nothing and the other reading costs every interface this + // instance owns. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + debug!("no VM directory yet; not collecting interfaces"); + return None; + } Err(error) => { warn!("failed to read the VM directory: {error}; not collecting interfaces"); return None; @@ -1005,88 +1013,104 @@ impl App { /// /// Per-VM release reaches only what its caller can still name. This reaches /// what nothing names any more, which is where a leak actually ends up: a - /// VM removed while the VMM was down, a workdir deleted by hand, a - /// teardown that raced a netd outage and was never retried because the VM - /// it belonged to no longer exists to retry it. + /// VM removed while the VMM was down, a workdir deleted by hand, a teardown + /// that raced a netd outage and was never retried because the VM it + /// belonged to no longer exists to retry it. + /// + /// The decision is made here and not in netd, and that is the whole design. + /// A collection decided in netd is decided against a set of live VMs that + /// was true when the caller sent it: netd runs the request when it wins the + /// operation lock, which may be long after, and a VM created in between is + /// absent from the set and present on the host. netd cannot close that -- + /// the lock that would close it is the VMM's per-VM launch lock, and netd + /// has no way to take it. Here, each VM is decided under exactly that lock + /// and re-checked while holding it, so a launch and a collection of the + /// same VM cannot both believe they are alone. /// - /// The live set is every VM this instance has, running or not. Not the + /// What is claimed is every VM this instance has, running or not. Not the /// running set: a VMM restarts under VMs that keep running, and collecting /// by what is running would delete their interfaces out from under them. - /// - /// At startup this must run before the API is served -- the set is a - /// snapshot, and a VM created after it was taken would be in netd's - /// listing and not in the snapshot. netd's own lock keeps the collection - /// from overlapping a prepare; the ordering here is what keeps it from - /// racing the decision. pub(crate) async fn reconcile_netd_interfaces(&self) { - let Some(live) = self.claimable_vm_ids() else { + let Some(claimed) = self.claimable_vm_ids() else { return; }; - let policy = if self.config.netd.collect_unattributed { - // The one decision worth a round trip before making it. Acting on - // the *absence* of an ownership record only means anything if the - // netd that built these interfaces writes one; asked of a netd that - // does not, "no record" describes every interface on the host -- - // including another instance's running VMs. - let reachability = self.netd_capabilities().await; - if reachability.records_ownership() { - netd::UnattributedPolicy::Remove - } else { - warn!( - netd = %reachability.describe(), - "netd.collect_unattributed is set, but this netd does not record which VM an \ - interface belongs to, so an interface with no record says nothing about what \ - is using it; collecting only what is attributed" - ); - netd::UnattributedPolicy::Keep - } - } else { - netd::UnattributedPolicy::Keep - }; - match netd::collect( - &self.config.netd.socket, - &self.config.cvm.instance_id, - live, - policy, - false, - ) - .await - { - Ok(collection) => { - if collection.removed > 0 { - for record in &collection.collected { - info!( - tap = %record.tap, - vm_id = record.vm_id.as_deref().unwrap_or("-"), - "collected a host interface no VM of this instance claims" - ); - } - info!( - removed = collection.removed, - "reconciled netd-managed interfaces" - ); - } - if collection.incomplete { - warn!("netd stopped collecting on its deadline; the next pass continues"); - } - } + let interfaces = match netd::list(&self.config.netd.socket, "").await { + Ok(interfaces) => interfaces, Err(error) if netd::is_unreachable(&error) => { - debug!("no netd to reconcile interfaces with") + debug!("no netd to reconcile interfaces with"); + return; } Err(error) => { - // Same reading a release gets: it answered, so now the question - // is whether the answer means it cannot do this at all. - if self.netd_capabilities().await.supports("gc") { - warn!("failed to reconcile netd-managed interfaces: {error:#}"); + // It answered, so ask whether the answer means it cannot do + // this at all -- the same reading a release gets. + if self.netd_capabilities().await.supports("list") { + warn!("failed to list netd-managed interfaces: {error:#}"); } else { warn!( - "netd on this node cannot collect interfaces no VM claims; a VM removed \ - while this VMM was down leaves its host interfaces behind until the node \ - runs a newer netd" + "netd on this node cannot say what it holds, so interfaces belonging to a \ + VM removed while this VMM was down stay until the node runs a newer netd" ); } + return; } + }; + let mut unattributed = 0; + let mut dead = BTreeSet::new(); + for record in &interfaces { + match (&record.instance_id, &record.vm_id) { + // Another instance's. Never ours to collect: on a host where + // two VMMs share one netd, this is the other one's running VM, + // and the ownership record is the only thing that says so. + (Some(instance_id), _) if instance_id != &self.config.cvm.instance_id => {} + (Some(_), Some(vm_id)) if !claimed.contains(vm_id) => { + dead.insert(vm_id.clone()); + } + (Some(_), _) => {} + // Built before netd recorded ownership, or by another netd. + // Nothing here can attribute it, so nothing here can decide + // about it: `dstack-vmm netd remove-interface` is where an + // operator who can decide says so. + _ => unattributed += 1, + } + } + if unattributed > 0 { + // Why there are any is the difference between "these are from + // before the upgrade and will sort themselves out" and "this netd + // never records ownership, so they never will". + let records_ownership = self.netd_capabilities().await.records_ownership(); + info!( + unattributed, + records_ownership, + "host interfaces carry no ownership record, so no VMM can tell whose they are \ + and none will collect them; `dstack-vmm netd list` shows them and \ + `netd remove-interface` removes one" + ); + } + for vm_id in dead { + // The lock a launch of this VM holds from before it asks netd for + // an interface until after it has recorded one. + let _launch = self.launch_lock(&vm_id).await; + // Re-read under it. Between the listing and this line the VM may + // have been created and started: its directory exists now, and its + // interfaces are the ones a launch just built. + if self.claims_vm(&vm_id) { + continue; + } + info!(vm_id = %vm_id, "collecting host interfaces no VM of this instance claims"); + self.release_vm_interfaces(&vm_id, &[]).await; + } + } + + /// Whether this instance has a VM by this ID at all, loaded or merely + /// present on disk. See [`App::claimable_vm_ids`]. + fn claims_vm(&self, vm_id: &str) -> bool { + if self.lock().vms.contains_key(vm_id) { + return true; } + // `validate_vm_id` keeps an ID from naming anything outside the VM + // directory, so this cannot be pointed at another path. + self.work_dir(vm_id) + .is_ok_and(|work_dir| work_dir.path().is_dir()) } /// Releases every host interface netd holds for this VM. @@ -2463,9 +2487,12 @@ mod tests { ); assert_eq!(app.claimable_vm_ids(), None); - // A VMM that has never run is a knowable answer, not an unknowable one. + // "Never ran" and "the volume is not mounted yet" produce the same + // error and only one of them means there are no VMs. A VMM that has + // never run has nothing to collect either way, so declining costs + // nothing and the other reading costs every interface it owns. let app = app_talking_to(Path::new("/nonexistent/netd.sock")); - assert_eq!(app.claimable_vm_ids(), Some(Vec::new())); + assert_eq!(app.claimable_vm_ids(), None); } /// A netd outage must not become a fleet that cannot be stopped. @@ -2505,9 +2532,9 @@ mod tests { } /// A probe in front of every stop is a second round trip on the hot path, - /// and -- because netd answers one caller at a time -- a *busy* netd would - /// answer it too slowly and be taken for an absent one, skipping the - /// release entirely. The operation cannot be misread that way. + /// and one whose failure mode is silence: a netd too slow to answer it + /// reads as an absent one, and an absent netd's release is skipped. The + /// operation itself cannot be misread that way. #[tokio::test] async fn a_netd_that_sweeps_is_asked_to_without_a_question_first() { let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ @@ -2542,59 +2569,107 @@ mod tests { ); } - /// A collection is decided against the set of VMs this instance has, and - /// asking for one from a netd that cannot do it must not look like asking - /// for one that found nothing. + fn held(tap: &str, instance: Option<&str>, vm: Option<&str>) -> serde_json::Value { + serde_json::json!({ + "tap": tap, + "kind": "tap", + "instance_id": instance, + "vm_id": vm, + "nic_index": 0, + "bound": false, + }) + } + + /// The decision a collection is made of. It lives here, and not in netd, + /// because it is only safe under a lock netd cannot take -- see + /// [`App::reconcile_netd_interfaces`]. + #[tokio::test] + async fn a_collection_takes_only_what_this_instance_no_longer_claims() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("live-vm")).unwrap(); + let netd = netd::testing::FakeNetd::spawn_holding( + netd::testing::Behavior::capable(&["hello", "list", "remove_all"]), + vec![ + held("dt000000000001", Some("test-instance"), Some("live-vm")), + held("dt000000000002", Some("test-instance"), Some("dead-vm")), + // Another VMM instance on the same host. The ownership record + // is the only thing that can tell this from ours, which is why + // there is one: without it a collection would delete another + // instance's running VM's networking. + held("dt000000000003", Some("someone-else"), Some("dead-vm")), + // Built before ownership was recorded, or by another netd. + // Nothing here can attribute it, so nothing here decides. + held("dt000000000004", None, None), + ], + ); + let app = App::new( + test_config(netd.socket(), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + app.reconcile_netd_interfaces().await; + + let swept: Vec = netd + .seen() + .into_iter() + .filter(|request| request["operation"] == "remove_all") + .collect(); + assert_eq!(swept.len(), 1, "one VM collected, and only one"); + assert_eq!(swept[0]["vm_id"], "dead-vm"); + assert_eq!(swept[0]["instance_id"], "test-instance"); + } + + /// A netd that cannot say what it holds must not read as one holding + /// nothing. #[tokio::test] - async fn reconciliation_is_asked_for_only_where_it_exists() { - let old = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ + async fn a_netd_that_cannot_enumerate_collects_nothing() { + let dir = tempfile::tempdir().unwrap(); + let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ "hello", "remove_all", ])); - let app = app_talking_to(old.socket()); - app.reconcile_netd_interfaces().await; - // Asked for, refused, and only then asked about -- so that "cannot do - // this" is reported as itself rather than as a collection that failed. - assert_eq!(old.operations(), vec!["gc", "hello"]); - - let netd = - netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&["hello", "gc"])); - let app = app_talking_to(netd.socket()); + let app = App::new( + test_config(netd.socket(), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); app.reconcile_netd_interfaces().await; - assert_eq!(netd.operations(), vec!["gc"]); - let request = &netd.seen()[0]; - assert_eq!(request["instance_id"], "test-instance"); - assert_eq!(request["live_vm_ids"].as_array().unwrap().len(), 0); - assert_eq!(request["dry_run"], false); - // The conservative default: an interface with no ownership record may - // be another VMM instance's running VM. - assert_eq!(request["unattributed"], "keep"); + assert!( + !netd + .operations() + .iter() + .any(|operation| operation == "remove_all"), + "nothing is collected on the strength of an answer netd could not give" + ); } + /// A VM directory that cannot be read is not an absent VM. Reading it as + /// one offers every interface this instance owns up for collection. #[tokio::test] - async fn collecting_unattributed_interfaces_is_the_operators_to_ask_for() { - let netd = - netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&["hello", "gc"])); - let mut config = test_config(netd.socket(), netd.socket().parent().unwrap()); - config.netd.collect_unattributed = true; - let app = App::new(config.clone(), SupervisorClient::new("http://127.0.0.1:0")); - app.reconcile_netd_interfaces().await; - assert_eq!(netd.operations(), vec!["hello", "gc"]); - assert_eq!(netd.seen()[1]["unattributed"], "remove"); - - // Acting on the absence of a record only means anything if the netd - // that built these interfaces writes one. Asked of a netd that does - // not, "no record" describes every interface on the host -- including - // another instance's running VMs. - let anonymous = - netd::testing::FakeNetd::spawn(netd::testing::Behavior::anonymous(&["hello", "gc"])); - config.netd.socket = anonymous.socket().to_path_buf(); - let app = App::new(config, SupervisorClient::new("http://127.0.0.1:0")); - app.reconcile_netd_interfaces().await; - assert_eq!(anonymous.seen()[1]["unattributed"], "keep"); + async fn an_unreadable_vm_directory_collects_nothing() { + let netd = netd::testing::FakeNetd::spawn_holding( + netd::testing::Behavior::capable(&["hello", "list", "remove_all"]), + vec![held( + "dt000000000001", + Some("test-instance"), + Some("running-vm"), + )], + ); + for run_path in [ + Path::new("/proc/self/environ"), + Path::new("/nonexistent/vms"), + ] { + let app = App::new( + test_config(netd.socket(), run_path), + SupervisorClient::new("http://127.0.0.1:0"), + ); + app.reconcile_netd_interfaces().await; + } + assert!( + netd.operations().is_empty(), + "an answer that could not be established is not an answer" + ); } - /// The window a launch spends between reading "not running" and actually + /// The window a launch spends between reading "not running" and actually /// The window a launch spends between reading "not running" and actually /// starting QEMU is long -- a GPU reset, a netd conversation -- and the /// sweep inside it deletes interfaces by deriving their names. Two entrants /// in that window meant the loser deleting the winner's live TAPs. diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 89695cf9e..194ae1a57 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -653,15 +653,6 @@ pub struct NetdConfig { /// not reach. Zero turns it off. #[serde(default = "default_reconcile_interval")] pub reconcile_interval_secs: u64, - /// Whether a collection may delete interfaces it cannot attribute. - /// - /// Off. An interface carrying no ownership record is not nobody's: on a - /// host where two VMM instances share one netd, it may be the other one's - /// running VM, and before netd recorded ownership every interface looked - /// like this. Turn it on only where nothing else creates interfaces in - /// netd's name space -- a single VMM instance on the host. - #[serde(default)] - pub collect_unattributed: bool, } /// See [`NetdConfig::reconcile_interval_secs`]. @@ -677,7 +668,6 @@ impl Default for NetdConfig { libvirt_uri: default_libvirt_uri(), network_filter: None, reconcile_interval_secs: default_reconcile_interval(), - collect_unattributed: false, } } } diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 8cdba5c45..4ecc4b7dc 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -99,6 +99,17 @@ enum NetdCommand { /// For a VM whose VMM will never ask again -- one whose directory was /// deleted by hand, or whose instance is gone. A running VMM collects /// these itself; this is for when there is no longer one to do it. + /// Delete one interface by name. + /// + /// For what nothing else can reach: an interface built before netd + /// recorded ownership, or by another netd, whose VM is gone. `netd list` + /// shows these with no instance and no VM -- nothing can attribute them, + /// so no VMM will ever collect them, and an operator who can tell what + /// they are says so here. + RemoveInterface { + /// The interface name, as `netd list` prints it. + name: String, + }, RemoveVm { /// The `cvm.instance_id` of the VMM that created them. `netd list` /// shows it. @@ -264,6 +275,13 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< } Ok(()) } + NetdCommand::RemoveInterface { name } => { + netd::remove_interface_named(&config.socket, name) + .await + .with_context(|| format!("failed to remove {name}"))?; + println!("removed {name}"); + Ok(()) + } NetdCommand::RemoveVm { instance, vm } => { let removed = netd::remove_all(&config.socket, instance, vm) .await diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index c6c7210ca..497699b25 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -58,7 +58,7 @@ const OPERATIONS: &[&str] = &[ "remove", "remove_all", "list", - "gc", + "remove_interface", "check", ]; /// The interface names netd may create. Reserved: anything matching it is @@ -174,30 +174,6 @@ pub struct IngressBinding { pub guest_port: u16, } -/// What a collection does about an interface it cannot attribute. -/// -/// Unattributed is not "nobody's". It is an interface built by a netd too old -/// to record ownership, by a third-party netd, or by this one in the instant -/// between creating an interface and recording it -- and on a host where two -/// VMM instances share a netd, one of those is another instance's running VM. -/// So the default is to leave it, and to say so. -/// -/// Nothing is stuck there: a collection still derives the names its own live -/// VMs would occupy and keeps those, so the interfaces of a running fleet -/// survive the upgrade that introduced the record, and each one gains a record -/// the next time its VM launches. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum UnattributedPolicy { - /// Leave it and report it. - #[default] - Keep, - /// Delete it. Only safe where nothing else creates interfaces in netd's - /// name space -- a node with a single VMM instance -- and the operator - /// says so. - Remove, -} - /// One host resource netd holds. /// /// `instance_id` and `vm_id` are absent when the interface carries no record @@ -358,27 +334,16 @@ pub enum Request { #[serde(default)] instance_id: String, }, - /// Delete every interface this VMM instance holds for a VM it no longer - /// has, and say what was left alone. + /// Delete one interface by name. /// - /// The one operation that does not need to be told what to look for. Per-VM - /// teardown reaches only what its caller can still name; this reaches - /// what nothing names any more, which is the only place a leak can end up. - /// - /// Releases everything those interfaces own, host ports included. See - /// [`PrepareBridgeRequest::ingress`]. - Gc { - instance_id: String, - /// Every VM this instance still has, running or not. Not the running - /// set: a VMM restarts under VMs that keep running, and collecting by - /// what is running would delete their interfaces out from under them. - live_vm_ids: Vec, - /// What to do with an interface carrying no ownership record. - #[serde(default)] - unattributed: UnattributedPolicy, - /// Report what would be collected without collecting it. - #[serde(default)] - dry_run: bool, + /// For what nothing else can reach: an interface built before netd recorded + /// ownership, or by another netd, whose VM is gone. Nothing can attribute + /// it, so nothing can decide about it -- but an operator looking at + /// `list` can, and this is how they say so. Guarded the same way every + /// other removal is: the name must be one netd could have created, and the + /// device must be one netd creates. + RemoveInterface { + tap: String, }, /// Verify a deterministic TAP and binding for operations and integration /// diagnostics. The VMM startup path uses Prepare rather than Check. @@ -447,13 +412,6 @@ enum Outcome { removed: usize, incomplete: bool, }, - /// A collection reports what it took as well as how much, because what it - /// took is the part an operator has to be able to disagree with. - Collected { - collected: Vec, - removed: usize, - incomplete: bool, - }, Hello(Capabilities), Listed(Vec), } @@ -500,15 +458,6 @@ impl Outcome { response.removed = Some(removed); response.incomplete = Some(incomplete); } - Self::Collected { - collected, - removed, - incomplete, - } => { - response.removed = Some(removed); - response.incomplete = Some(incomplete); - response.interfaces = Some(collected); - } Self::Hello(capabilities) => response.capabilities = Some(capabilities), Self::Listed(interfaces) => response.interfaces = Some(interfaces), } @@ -665,6 +614,14 @@ pub async fn remove_all(socket: &Path, instance_id: &str, vm_id: &str) -> Result .context("netd answered a sweep without saying what it removed") } +/// Deletes one interface by name. See [`Request::RemoveInterface`]. +pub async fn remove_interface_named(socket: &Path, tap: &str) -> Result<()> { + let request = Request::RemoveInterface { + tap: tap.to_string(), + }; + exchange(socket, &request).await.map(|_| ()) +} + /// Everything netd holds, optionally narrowed to one VMM instance. See /// [`Request::List`]. pub async fn list(socket: &Path, instance_id: &str) -> Result> { @@ -677,39 +634,6 @@ pub async fn list(socket: &Path, instance_id: &str) -> Result, - pub incomplete: bool, -} - -/// Deletes every interface this VMM instance holds for a VM it no longer has. -/// See [`Request::Gc`]. -pub async fn collect( - socket: &Path, - instance_id: &str, - live_vm_ids: Vec, - unattributed: UnattributedPolicy, - dry_run: bool, -) -> Result { - let request = Request::Gc { - instance_id: instance_id.to_string(), - live_vm_ids, - unattributed, - dry_run, - }; - let response = exchange(socket, &request).await?; - Ok(Collection { - removed: response - .removed - .context("netd answered a collection without saying what it removed")?, - collected: response.interfaces.unwrap_or_default(), - incomplete: response.incomplete.unwrap_or_default(), - }) -} - /// This node's netd, as far as it can be asked. #[derive(Debug, Clone)] pub enum Reachability { @@ -735,15 +659,15 @@ impl Reachability { } } - /// Whether asking this netd to forward host ports is meaningful. Unknown - /// counts as no: a caller that assumed yes would report ports as published - /// on the strength of never having asked. /// Whether it records which VM an interface belongs to. Unknown counts as /// no, for the same reason `forwards_ingress` does. pub fn records_ownership(&self) -> bool { matches!(self, Self::Capable(capabilities) if capabilities.attribution) } + /// Whether asking this netd to forward host ports is meaningful. Unknown + /// counts as no: a caller that assumed yes would report ports as published + /// on the strength of never having asked. pub fn forwards_ingress(&self) -> bool { matches!(self, Self::Capable(capabilities) if capabilities.ingress) } @@ -806,7 +730,7 @@ async fn exchange(socket: &Path, request: &Request) -> Result { Request::Remove { .. } => "remove", Request::RemoveAll { .. } => "remove_all", Request::List { .. } => "list", - Request::Gc { .. } => "gc", + Request::RemoveInterface { .. } => "remove_interface", Request::Check { .. } => "check", }; let exchange = async { @@ -1045,18 +969,13 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { incomplete, }) } - Request::Gc { - instance_id, - live_vm_ids, - unattributed, - dry_run, - } => collect_garbage( - libvirt_uri, - &instance_id, - live_vm_ids, - unattributed, - dry_run, - ), + Request::RemoveInterface { tap } => { + if !is_managed_name(&tap) { + bail!("{tap} is not a name netd could have created"); + } + remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; + Ok(Outcome::tap(tap)) + } Request::Remove { identity, filtered } => { validate_identity(&identity)?; let tap = tap_name(&identity); @@ -1308,7 +1227,13 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res }; validate_identity(&identity)?; let bindings = existing_bindings(libvirt_uri); - let mut libvirt = bindings.is_some(); + // Not `bindings.is_some()`. A listing that could not be produced says + // nothing about whether a *deletion* will work, and reading it as "libvirt + // is down, skip the bindings" would mean a node whose listing breaks for + // any reason silently stops cleaning up bindings at all -- which is worse + // than the per-name asking this listing exists to avoid. The pass finds + // out by trying, once. + let mut libvirt = true; let mut removed = 0; let mut incomplete = false; let mut first_error = None; @@ -1359,161 +1284,6 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res } } -/// What a collection would take, decided against a listing rather than against -/// the host. -/// -/// Split out so it can be reasoned about and tested without a privileged -/// daemon: the decision is the dangerous part, not the `ip link delete` that -/// follows it. -fn gc_plan( - records: &[InterfaceRecord], - instance_id: &str, - live_vm_ids: &HashSet, - live_names: &dyn Fn() -> HashSet, - policy: UnattributedPolicy, -) -> Vec { - let mut derived = None; - let mut collected = Vec::new(); - for record in records { - let keep = match (&record.instance_id, &record.vm_id) { - // Someone else's, and on a host where two VMM instances share one - // netd, "someone else's" includes a running VM. The record is the - // only thing that can tell them apart, which is why there is one. - (Some(owner), _) if owner != instance_id => true, - (Some(_), Some(vm_id)) => live_vm_ids.contains(vm_id), - // Cannot be attributed. Derive the names this instance's live VMs - // would occupy and keep those, so a fleet that was running before - // ownership was recorded survives the upgrade that introduced it. - _ => { - let names = derived.get_or_insert_with(live_names); - names.contains(&record.tap) - // An nwfilter binding whose interface is gone is the one - // unattributable thing that is also unambiguously dead: a - // record can only ever live on an interface, so this can - // never gain one, and nothing is using a binding with - // nothing to bind to. Left to the conservative default it - // would be the one leak in this design that nothing could - // ever collect -- the VM it belonged to is gone, so not - // even an operator could name it. - || (record.kind != "binding" && policy == UnattributedPolicy::Keep) - } - }; - if !keep { - collected.push(record.clone()); - } - } - collected -} - -/// Every interface name this instance's live VMs could occupy. -/// -/// Only ever needed when something on the host carries no ownership record, so -/// it is derived lazily: |live| x 256 digests is cheap next to a `virsh` call -/// but not next to nothing. -fn live_interface_names(instance_id: &str, live_vm_ids: &HashSet) -> HashSet { - let mut names = HashSet::with_capacity(live_vm_ids.len() * (MAX_NIC_INDEX + 1)); - for vm_id in live_vm_ids { - for nic_index in 0..=MAX_NIC_INDEX { - names.insert(tap_name(&InterfaceIdentity { - instance_id: instance_id.to_string(), - vm_id: vm_id.clone(), - nic_index, - })); - } - } - names -} - -/// Deletes what [`gc_plan`] decided against. See [`Request::Gc`]. -fn collect_garbage( - libvirt_uri: &str, - instance_id: &str, - live_vm_ids: Vec, - policy: UnattributedPolicy, - dry_run: bool, -) -> Result { - if instance_id.is_empty() || instance_id.len() > 128 || instance_id.contains(':') { - bail!("invalid instance ID"); - } - let live: HashSet = live_vm_ids.into_iter().collect(); - let records = list_interfaces(libvirt_uri, ""); - let unattributed = records - .iter() - .filter(|record| record.instance_id.is_none()) - .count(); - let collected = gc_plan( - &records, - instance_id, - &live, - &|| live_interface_names(instance_id, &live), - policy, - ); - if dry_run { - info!( - %instance_id, - held = records.len(), - unattributed, - would_remove = collected.len(), - "collection dry run" - ); - return Ok(Outcome::Collected { - removed: 0, - incomplete: false, - collected, - }); - } - let mut libvirt = true; - let mut removed = 0; - let mut incomplete = false; - let mut taken = Vec::new(); - let deadline = std::time::Instant::now() + COLLECTION_DEADLINE; - for record in collected { - if std::time::Instant::now() >= deadline { - warn!("collection stopped on its deadline"); - incomplete = true; - break; - } - let outcome = if record.kind == "binding" { - // No interface left to delete, only the binding that outlived it. - delete_binding(libvirt_uri, &record.tap) - } else { - remove_interface_in_pass(libvirt_uri, &record.tap, &mut libvirt) - }; - match outcome { - Ok(()) => { - info!( - tap = %record.tap, - vm_id = record.vm_id.as_deref().unwrap_or("-"), - "collected interface no live VM claims" - ); - removed += 1; - taken.push(record); - } - // Keep going: one stuck interface must not shelter the rest. - Err(error) => warn!(tap = %record.tap, "failed to collect: {error:#}"), - } - } - let left_alone = unattributed.saturating_sub( - taken - .iter() - .filter(|record| record.instance_id.is_none()) - .count(), - ); - if left_alone > 0 && policy == UnattributedPolicy::Keep { - info!( - %instance_id, - unattributed = left_alone, - "interfaces carry no ownership record and were left alone; each gains one when its VM \ - next launches" - ); - } - Ok(Outcome::Collected { - removed, - incomplete, - collected: taken, - }) -} - /// Every resource netd owns, read off the host rather than out of a record. /// /// Ownership is the reserved name plus the kernel's own answer about what kind @@ -1540,7 +1310,7 @@ fn list_interfaces(libvirt_uri: &str, instance_id: &str) -> Vec "tap" } else { // The name is netd's to use, but this is not a device netd - // creates. Listing it would invite a collection to delete it. + // creates. Listing it would invite a caller to delete it. continue; }; let alias = @@ -1560,11 +1330,11 @@ fn list_interfaces(libvirt_uri: &str, instance_id: &str) -> Vec }); } } - // A binding outlives its interface, and an interface is the only thing - // that carries a record, so an orphaned binding can never be attributed. - // It is still netd's: nothing else creates a binding at one of these names. + // A binding outlives its interface, and an interface is the only thing that + // carries a record, so an orphaned binding can never be attributed. It is + // still netd's: nothing else creates a binding at one of these names. for name in bindings.into_iter().flatten() { - if is_managed_name(&name) && !seen.contains(&name) { + if !seen.contains(&name) { records.push(InterfaceRecord { tap: name, kind: "binding".to_string(), @@ -1619,9 +1389,6 @@ fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Re Ok(()) } -/// Whether this is a tun/tap device. `tun_flags` is published by the tun -/// driver and by nothing else, so its presence is the kernel's own answer -- -/// as `macvtap/` is for the other kind of device netd creates. /// Records who an interface belongs to, on the interface. See /// [`interface_alias`]. fn set_alias(tap: &str, identity: &InterfaceIdentity) -> Result<()> { @@ -1630,6 +1397,9 @@ fn set_alias(tap: &str, identity: &InterfaceIdentity) -> Result<()> { .with_context(|| format!("failed to record ownership on {tap}")) } +/// Whether this is a tun/tap device. `tun_flags` is published by the tun +/// driver and by nothing else, so its presence is the kernel's own answer -- +/// as `macvtap/` is for the other kind of device netd creates. fn is_tuntap(interface: &str) -> bool { Path::new("/sys/class/net") .join(interface) @@ -1647,8 +1417,9 @@ fn is_macvtap(interface: &str) -> bool { /// Deletes an interface's nwfilter binding, if it has one. /// /// Goes through the same `COMMAND_TIMEOUT`-bounded helper as every other virsh -/// call. netd's accept loop is strictly serialized, so an unbounded call here -/// would let one unreachable libvirt stall every other VM's prepare and remove. +/// call. Every mutating operation holds the operation lock, so an unbounded +/// call here would let one unreachable libvirt stall every other VM's prepare +/// and remove behind it. fn delete_binding(uri: &str, tap: &str) -> Result<()> { match virsh(uri, &["nwfilter-binding-delete", tap], None) { Ok(()) => Ok(()), @@ -1844,15 +1615,29 @@ fn virsh_output(uri: &str, args: &[&str], stdin: Option<&[u8]>) -> Result Option> { - match virsh_output(uri, &["nwfilter-binding-list", "--name"], None) { - Ok(output) => Some(output.split_whitespace().map(str::to_string).collect()), + match virsh_output(uri, &["nwfilter-binding-list"], None) { + Ok(output) => Some( + output + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|name| is_managed_name(name)) + .map(str::to_string) + .collect(), + ), Err(error) => { debug!("could not list nwfilter bindings: {error:#}"); None @@ -2008,17 +1793,28 @@ pub(crate) mod testing { impl FakeNetd { pub(crate) fn spawn(behavior: Behavior) -> Self { + Self::spawn_holding(behavior, Vec::new()) + } + + /// A netd that holds these interfaces, whatever else it does. + pub(crate) fn spawn_holding(behavior: Behavior, interfaces: Vec) -> Self { + Self::start(behavior, interfaces) + } + + fn start(behavior: Behavior, interfaces: Vec) -> Self { let dir = tempfile::tempdir().expect("tempdir"); let socket = dir.path().join("netd.sock"); let listener = UnixListener::bind(&socket).expect("bind"); let seen = Arc::new(Mutex::new(Vec::new())); let recorder = seen.clone(); + let interfaces = std::sync::Arc::new(interfaces); tokio::spawn(async move { loop { let Ok((mut stream, _)) = listener.accept().await else { return; }; let behavior = behavior.clone(); + let interfaces = interfaces.clone(); let recorder = recorder.clone(); tokio::spawn(async move { let mut message = Vec::new(); @@ -2029,7 +1825,7 @@ pub(crate) mod testing { return; }; recorder.lock().expect("poisoned").push(request.clone()); - let response = answer(&behavior, &request); + let response = answer(&behavior, &interfaces, &request); let _ = stream .write_all(&serde_json::to_vec(&response).unwrap()) .await; @@ -2061,7 +1857,7 @@ pub(crate) mod testing { } } - fn answer(behavior: &Behavior, request: &Value) -> Value { + fn answer(behavior: &Behavior, interfaces: &[Value], request: &Value) -> Value { let operation = request["operation"].as_str().unwrap_or_default(); let (operations, ingress, attribution) = match behavior { Behavior::Legacy => { @@ -2112,8 +1908,18 @@ pub(crate) mod testing { } "remove" | "check" => json!({"ok": true, "tap": "dtdeadbeef00"}), "remove_all" => json!({"ok": true, "removed": 0, "incomplete": false}), - "gc" => json!({"ok": true, "removed": 0, "incomplete": false, "interfaces": []}), - "list" => json!({"ok": true, "interfaces": []}), + "list" => { + let instance = request["instance_id"].as_str().unwrap_or_default(); + let held: Vec = interfaces + .iter() + .filter(|record| { + instance.is_empty() || record["instance_id"].as_str() == Some(instance) + }) + .cloned() + .collect(); + json!({"ok": true, "interfaces": held}) + } + "remove_interface" => json!({"ok": true, "tap": request["tap"]}), _ => json!({"ok": true}), } } @@ -2716,105 +2522,29 @@ mod tests { assert!(tap_name(&identity("instance", "vm", 255)).len() < 16); } - fn record(tap: &str, instance: Option<&str>, vm: Option<&str>) -> InterfaceRecord { - InterfaceRecord { - tap: tap.to_string(), - kind: "tap".to_string(), - instance_id: instance.map(str::to_string), - vm_id: vm.map(str::to_string), - nic_index: instance.map(|_| 0), - bound: false, - } - } - - fn plan(records: &[InterfaceRecord], live: &[&str], policy: UnattributedPolicy) -> Vec { - let live: HashSet = live.iter().map(|vm| vm.to_string()).collect(); - gc_plan( - records, - "ours", - &live, - &|| live_interface_names("ours", &live), - policy, - ) - .into_iter() - .map(|record| record.tap) - .collect() - } - - /// The decision a collection is made of, which is the dangerous half: the - /// `ip link delete` that follows it is not the part that can take down a - /// running VM. + /// The command prints a table for a human and accepts no options to make it + /// print anything else, so this parses one. Narrowing to netd's own name + /// space is what makes that safe. #[test] - fn a_collection_takes_only_what_this_instance_no_longer_claims() { - let records = [ - record("dt000000000001", Some("ours"), Some("live-vm")), - record("dt000000000002", Some("ours"), Some("dead-vm")), - // Another VMM instance on the same host. The record is the only - // thing that can tell this from ours, which is why there is one: - // without it a collection would delete another instance's running - // VM's networking. - record("dt000000000003", Some("theirs"), Some("dead-vm")), - ]; - assert_eq!( - plan(&records, &["live-vm"], UnattributedPolicy::Keep), - vec!["dt000000000002"] - ); - // A VM this instance no longer has at all is exactly the case per-VM - // teardown can never reach: nothing is left to name it. - assert_eq!( - plan(&records, &[], UnattributedPolicy::Keep).len(), - 2, - "both of ours, and never theirs" - ); - } - - /// The upgrade this has to survive: before ownership was recorded, every - /// interface on the host was unattributed, and a collection that deleted - /// them would take down the networking of every running VM at once. - #[test] - fn a_fleet_that_predates_the_ownership_record_survives_the_first_collection() { - let live_tap = tap_name(&identity("ours", "live-vm", 0)); - let records = [ - record(&live_tap, None, None), - record("dt0000000000ff", None, None), - ]; - - // The conservative default leaves both, and says so. - assert!(plan(&records, &["live-vm"], UnattributedPolicy::Keep).is_empty()); - - // Even told to collect, a name a live VM would occupy is kept: the - // record is missing, but the name is still derivable from the identity, - // and that is proof enough to keep something. - assert_eq!( - plan(&records, &["live-vm"], UnattributedPolicy::Remove), - vec!["dt0000000000ff".to_string()] - ); - } - - /// A binding outlives the interface it was bound to, so it is the one piece - /// of state that can never carry a record -- and therefore the one thing a - /// conservative default would strand forever. It is also the one - /// unattributable thing that is unambiguously dead: nothing is using a - /// binding with nothing to bind to, and the VM it belonged to is gone, so - /// not even an operator could name it to remove it by hand. - #[test] - fn an_orphaned_binding_is_collected_even_by_default() { - let mut binding = record("dt00000000beef", None, None); - binding.kind = "binding".to_string(); - binding.bound = true; - let records = [binding]; - assert_eq!( - plan(&records, &[], UnattributedPolicy::Keep), - vec!["dt00000000beef".to_string()] - ); - - // Except where the name is one a live VM would occupy: a filtered - // interface is rebuilt binding-first, so the binding can legitimately - // exist for a moment without one. - let live_tap = tap_name(&identity("ours", "live-vm", 0)); - let mut binding = record(&live_tap, None, None); - binding.kind = "binding".to_string(); - assert!(plan(&[binding], &["live-vm"], UnattributedPolicy::Keep).is_empty()); + fn the_binding_listing_reads_a_table_meant_for_a_person() { + let output = "\ + Port Dev Filter +--------------------------------- + dt1e053266e9f7 clean-traffic + dt28b105b3031a clean-traffic + vnet3 some-other-filter +"; + let names: HashSet = output + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|name| is_managed_name(name)) + .map(str::to_string) + .collect(); + assert_eq!(names.len(), 2); + assert!(names.contains("dt1e053266e9f7")); + // The header, the rule, and a binding that is not netd's all fall out. + assert!(!names.contains("Port")); + assert!(!names.contains("vnet3")); } /// Everything else here reasons about strings. This puts the reasoning @@ -2914,7 +2644,6 @@ mod tests { // and most of them miss. remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); } - /// A netd that answers a sweep with no count did not sweep. Reading the /// absent field as zero is the same conflation `queues` and `ingress` are /// shaped to avoid, and here it would report a netd that cannot collect a diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index f0b7abd0f..eb3875b14 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -158,12 +158,7 @@ socket = "/run/dstack/netd.sock" # the VMM knows every VM it has and everything a crash left behind is still # there. This is the backstop for what accumulates while it runs. 0 disables it. reconcile_interval_secs = 3600 -# Whether a collection may delete interfaces carrying no ownership record. An -# unattributed interface is not nobody's: where two VMM instances share one -# netd it may be the other one's running VM, and before netd recorded ownership -# every interface looked like this. Turn it on only on a node with a single VMM -# instance, where nothing else creates interfaces in netd's name space. -collect_unattributed = false + # Applied when netd creates the socket itself. A systemd socket unit controls # its own SocketMode instead. socket_mode = 0o660 From 2a7cf696790ed153502611f998f04fa3ee2a0970 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 03:41:57 -0700 Subject: [PATCH 17/34] fix(vmm): do not cache an answer that may have been a blip `Unreachable` and `Legacy` are both produced by transient failures, and holding either for half a minute turns one blip into a deployment refused because 'this netd does not forward host ports' -- which may not be true. Only a real capability answer is worth reusing; the other two are re-asked on paths that were already making a round trip. --- docs/bridge-networking.md | 6 ++++++ dstack/vmm/src/app.rs | 45 ++++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index c685cefd7..d70d1c96d 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -304,6 +304,12 @@ sudo dstack-vmm netd remove-interface dtc41d9e0b7a52 Nothing accumulates: each interface gains a record the next time its VM launches, so the set only shrinks. +Changing `cvm.instance_id` — or `run_path`, which it is derived from — is the +one move that strands interfaces on purpose. They stay recorded under the old +namespace, so no VMM collects them and running VMs keep working until they +stop. `netd list` still shows the old instance ID, which is what +`netd remove-vm --instance --vm ` needs. + ### Mixing networking modes Bridge and user-mode VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index a4282da9a..0feef04f3 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -931,7 +931,13 @@ impl App { return reachability.clone(); } let reachability = netd::probe(&self.config.netd.socket).await; - if reachability.is_reachable() { + // Only a real answer is cached. "Unreachable" and "too old to answer" + // are both produced by transient failures too, and holding either for + // half a minute turns one blip into a deployment refused for a reason + // that is not true -- `refuse_unpublishable_ports` reads this. A netd + // that genuinely predates `hello` is re-asked on a path that was + // already making a round trip. + if matches!(reachability, netd::Reachability::Capable(_)) { *self.netd_probe.lock().or_panic("mutex poisoned") = Some((std::time::Instant::now(), reachability.clone())); } @@ -941,11 +947,11 @@ impl App { /// Every VM whose interfaces this instance may still be using. /// /// Wider than the VMs it managed to load. A VM whose manifest is corrupt - /// or whose image is missing fails to load and is only logged -- but its - /// QEMU may well still be running, and collecting by what loaded would - /// delete a running VM's networking over a file the VMM could not parse. - /// A directory is enough of a claim: what has been removed for real leaves - /// none behind. + /// or whose image is missing fails to load and is only logged; `reload_vms` + /// then stops any supervisor process it still has, but that runs in the + /// background, and a collection deciding on the loaded set alone races it + /// -- over a file the VMM could not parse. A directory is enough of a + /// claim, and what has been removed for real leaves none behind. /// `None` when the answer cannot be established, which is not the same as /// nobody claiming anything: an unreadable VM directory read as empty /// would offer every interface on the host up for collection. @@ -2554,18 +2560,27 @@ mod tests { assert!(sweep.get("nic_index").is_none()); } - /// Where a probe is unavoidable it covers a window rather than one call, - /// or netd's accept loop services a connection per VM per operation. + /// A real answer is reused; anything else is asked again. + /// + /// "Unreachable" and "too old to answer" are both produced by transient + /// failures too, and holding either for half a minute turns one blip into + /// a deployment refused for a reason that is not true. #[tokio::test] - async fn the_capability_answer_is_reused() { - let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); + async fn only_a_real_capability_answer_is_reused() { + let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&["hello"])); let app = app_talking_to(netd.socket()); - app.release_vm_interfaces("vm-1", &[]).await; - app.release_vm_interfaces("vm-2", &[]).await; + app.netd_capabilities().await; + app.netd_capabilities().await; + assert_eq!(netd.operations(), vec!["hello"], "asked once, read twice"); + + let legacy = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); + let app = app_talking_to(legacy.socket()); + app.netd_capabilities().await; + app.netd_capabilities().await; assert_eq!( - netd.operations().iter().filter(|op| *op == "hello").count(), - 1, - "asked once, acted on twice" + legacy.operations().len(), + 2, + "an answer that may have been a blip is not held onto" ); } From 93571ec999daaaabcd4d11e03d86773c55091051 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 03:42:41 -0700 Subject: [PATCH 18/34] chore(netd): drop a test helper the collection reshape left unused --- dstack/vmm/src/netd.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 497699b25..a9dbf736d 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -1766,23 +1766,6 @@ pub(crate) mod testing { other => other, } } - - /// A netd that builds interfaces without recording whose they are -- - /// a third-party one, or this one before it did. - pub(crate) fn anonymous(operations: &[&str]) -> Self { - match Self::capable(operations) { - Self::Capable { - operations, - ingress, - .. - } => Self::Capable { - operations, - ingress, - attribution: false, - }, - other => other, - } - } } pub(crate) struct FakeNetd { From 4032696bd603cb7e0aa2b1982d042bd310bc0d4a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 04:25:43 -0700 Subject: [PATCH 19/34] fix(vmm): decide whether a VM is running under the lock that keeps it still An audit of the stop-time removal found the removal itself sound and the *decisions* around it not. Two callers read "is it running" outside the per-VM launch lock and acted on the answer inside it, which is precisely the race the lock exists to close. `update_vm` with a networking change: the read says not-running, a launch then takes the lock, prepares TAPs and deploys QEMU, and the update takes the lock afterwards and releases the interfaces of a VM that is now running. QEMU keeps running with a dead NIC, the supervisor still reports it healthy, and nothing logs a thing. `finish_remove_vm`: it stopped the process and waited for exit entirely outside the lock. A launch that passed the `removing` check before the marker was set is already inside the lock and has not deployed yet, so the wait sees nothing running and returns at once; the launch then starts QEMU, and removal takes the lock and deletes its interfaces, its workdir and its CID. The VM runs on, invisible, until the next orphan sweep at startup. Both now take the lock first and ask afterwards. A third, smaller: an automatic restart reads the started flag off disk and only then queues a launch, which waits for the lock a concurrent stop is holding. Without a re-read under that lock, the launch resurrects a VM the operator was told was stopped. An explicit start sets the flag itself and has nothing to re-read, so only the automatic path re-checks. Two things the same audit found in the removal proper. A sweep truncated by netd's deadline reported `incomplete`, which the client dropped -- so a host with interfaces left over logged "released 3 interfaces" and nothing else; it now says so. And `Request::Remove` no longer fails when libvirt cannot delete a binding: that strictness was written for prepare, where a leftover binding blocks the creation about to happen, and at removal it only leaves the interface itself up on the bridge instead of a binding libvirt hands back on its next listing. It also now agrees with the whole-VM sweep, which was always best effort. The comment on `stop_vm` claimed reconciliation would collect what a failed release left behind. It will not: a stopped VM is still one this instance claims, so a VM that is never started or removed again keeps its interfaces. Say that instead. --- dstack/vmm/src/app.rs | 87 ++++++++++++++++++++++++++++++---- dstack/vmm/src/main.rs | 7 ++- dstack/vmm/src/main_service.rs | 10 ++-- dstack/vmm/src/netd.rs | 47 +++++++++++++----- 4 files changed, 125 insertions(+), 26 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 0feef04f3..d5fe622b3 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -485,6 +485,14 @@ impl App { } } } + // A restart decided before a stop must not outlive it. The decision + // read `started` from disk; `stop_vm` writes it false under this lock, + // so re-reading it here is what makes the stop stick. An explicit start + // sets the flag itself and has nothing to re-read. + if !reset_restart_policy && !self.work_dir(id)?.started().unwrap_or(false) { + debug!(id, "skipping automatic restart: the VM was stopped"); + return Ok(()); + } self.sync_dynamic_config(id)?; let is_running = self .supervisor @@ -603,9 +611,11 @@ impl App { self.stop_vm_process(id).await?; let networks = self.work_dir(id)?.runtime_networks(); // Not fallible: a VM that has been asked to stop is stopped whether or - // not netd could be reached, and what is left behind is collected by - // the next launch or by reconciliation. See - // [`App::release_vm_interfaces`]. + // not netd could be reached. What is left behind is reclaimed by this + // VM's next launch, which releases before it prepares, or by its + // removal. Not by reconciliation -- a stopped VM is still one this + // instance claims, so a VM that is never started or removed again keeps + // its interfaces. See [`App::release_vm_interfaces`]. self.release_vm_interfaces(id, &networks).await; Ok(()) } @@ -1147,8 +1157,26 @@ impl App { ) .await { - Ok(0) => {} - Ok(removed) => info!(vm_id, removed, "released netd-managed interfaces"), + Ok(sweep) => { + if sweep.removed > 0 { + info!( + vm_id, + removed = sweep.removed, + "released netd-managed interfaces" + ); + } + // A sweep that ran out of time is not a sweep that found + // nothing left. Saying so is the difference between a host an + // operator can reason about and one where "released 3 + // interfaces" hid the fourth. + if sweep.incomplete { + warn!( + vm_id, + "netd stopped releasing this VM's interfaces on its deadline; the rest \ + are released by its next launch or its removal" + ); + } + } Err(error) if netd::is_unreachable(&error) => { debug!(vm_id, %error, "no netd to release interfaces from") } @@ -1276,6 +1304,15 @@ impl App { /// /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup. async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> { + // Held across the stop, the wait and the release, not just the release. + // `removing` turns launches away, but a launch that passed that check + // before the marker was set is already inside the lock: it has not + // deployed yet, so the wait below sees nothing running and returns at + // once, and the release then deletes the interfaces of the QEMU that + // launch went on to start. Taking the lock first means the launch + // finishes before removal decides anything, and removal then stops what + // it actually started. + let _launch = self.launch_lock(id).await; // Stop the supervisor process (idempotent if already stopped) if let Err(err) = self.stop_vm_process(id).await { debug!("graceful VM stop during removal failed: {err:?}"); @@ -1315,10 +1352,7 @@ impl App { } let runtime_networks = self.work_dir(id)?.runtime_networks(); - { - let _launch = self.launch_lock(id).await; - self.release_vm_interfaces(id, &runtime_networks).await; - } + self.release_vm_interfaces(id, &runtime_networks).await; // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. @@ -2684,7 +2718,40 @@ mod tests { ); } - /// The window a launch spends between reading "not running" and actually /// The window a launch spends between reading "not running" and actually + /// A restart decided before a stop must not outlive it. The restart task + /// reads the started flag off disk and only then queues a launch, which + /// waits for the lock the stop is holding; without a re-read under that + /// lock the launch resurrects a VM the operator was told was stopped. + #[tokio::test] + async fn an_automatic_restart_does_not_outlive_the_stop_it_raced() { + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(Path::new("/nonexistent/netd.sock"), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + let work_dir = app.work_dir("vm-1").unwrap(); + std::fs::create_dir_all(work_dir.path()).unwrap(); + + work_dir.set_started(false).unwrap(); + app.start_vm_with_restart_policy("vm-1", false) + .await + .expect("an automatic restart of a stopped VM does nothing"); + + // The flag is the whole difference: with it set, the same call goes on + // to do the work, and fails here for want of a VM to launch. + work_dir.set_started(true).unwrap(); + assert!(app + .start_vm_with_restart_policy("vm-1", false) + .await + .is_err()); + + // An explicit start sets the flag itself and has nothing to re-read, + // so it is never turned away by one. + work_dir.set_started(false).unwrap(); + assert!(app.start_vm("vm-1").await.is_err()); + } + + /// The window a launch spends between reading "not running" and actually /// starting QEMU is long -- a GPU reset, a netd conversation -- and the /// sweep inside it deletes interfaces by deriving their names. Two entrants /// in that window meant the loser deleting the winner's live TAPs. diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 4ecc4b7dc..5af0cc5de 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -283,10 +283,13 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< Ok(()) } NetdCommand::RemoveVm { instance, vm } => { - let removed = netd::remove_all(&config.socket, instance, vm) + let sweep = netd::remove_all(&config.socket, instance, vm) .await .context("failed to remove the VM's interfaces")?; - println!("removed {removed} interface(s) for {vm}"); + println!("removed {} interface(s) for {vm}", sweep.removed); + if sweep.incomplete { + println!("netd stopped on its deadline; run this again to continue"); + } Ok(()) } } diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 9d7f76e12..7ec47e17d 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -1068,6 +1068,13 @@ impl VmmRpc for RpcHandler { let networks = networks_from_proto(&request.networks, &cvm)?; resolve_requested_networks(&networks, &cvm, manifest.vcpu)? }; + // The lock first, then the question. Reading "not running" outside + // it and acting on the answer inside is the exact race the lock + // exists to close: a launch can start, prepare its interfaces and + // deploy QEMU in between, and the release would then delete the + // interfaces of a VM that is running -- silently, since QEMU stays + // up and the supervisor still reports it healthy. + let _launch = self.app.launch_lock(&request.id).await; let is_running = self .app .supervisor @@ -1076,9 +1083,6 @@ impl VmmRpc for RpcHandler { .is_some_and(|info| info.state.status.is_running()); if !is_running { let runtime_networks = vm_work_dir.runtime_networks(); - // Teardown deletes interfaces by deriving their names, so it - // must not overlap a launch of the same VM. - let _launch = self.app.launch_lock(&request.id).await; self.app .release_vm_interfaces(&request.id, &runtime_networks) .await; diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index a9dbf736d..71d7bfa9d 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -603,15 +603,31 @@ pub async fn request(socket: &Path, request: &Request) -> Result Result { +pub async fn remove_all(socket: &Path, instance_id: &str, vm_id: &str) -> Result { let request = Request::RemoveAll { instance_id: instance_id.to_string(), vm_id: vm_id.to_string(), }; - exchange(socket, &request) - .await? - .removed - .context("netd answered a sweep without saying what it removed") + let response = exchange(socket, &request).await?; + Ok(Sweep { + removed: response + .removed + .context("netd answered a sweep without saying what it removed")?, + // Absent from a netd that cannot stop early, which is the same as one + // that did not. + incomplete: response.incomplete.unwrap_or_default(), + }) +} + +/// What one whole-VM sweep did. +#[derive(Debug, Clone, Copy)] +pub struct Sweep { + pub removed: usize, + /// Whether it stopped on its deadline with names left to check. Reported + /// rather than dropped: a sweep that ran out of time and one that finished + /// having removed the same count are different states of the host, and + /// only one of them needs looking at. + pub incomplete: bool, } /// Deletes one interface by name. See [`Request::RemoveInterface`]. @@ -976,10 +992,20 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; Ok(Outcome::tap(tap)) } - Request::Remove { identity, filtered } => { + Request::Remove { + identity, + filtered: _, + } => { validate_identity(&identity)?; let tap = tap_name(&identity); - remove_interface(libvirt_uri, &tap, binding_cleanup(filtered))?; + // Best effort, whatever the caller says was built. The strict rule + // exists for prepare, where a binding left at the name would block + // the one about to be created; at removal nothing is about to take + // the name, and failing here leaves the interface itself up on the + // bridge rather than just a binding libvirt will hand back on its + // next listing. It also makes this agree with the whole-VM sweep, + // which has always been best effort. + remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; Ok(Outcome::tap(tap)) } Request::Check { identity, filtered } => { @@ -2641,10 +2667,9 @@ mod tests { assert!(!is_unreachable(&error)); let netd = testing::FakeNetd::spawn(testing::Behavior::capable(&["hello", "remove_all"])); - assert_eq!( - remove_all(netd.socket(), "instance", "vm").await.unwrap(), - 0 - ); + let sweep = remove_all(netd.socket(), "instance", "vm").await.unwrap(); + assert_eq!(sweep.removed, 0); + assert!(!sweep.incomplete); } /// Every "an unreachable netd is not a failure" branch in the VMM hangs From a920aa6d45d47ef86bfc274119db9a5cc49a605a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 04:34:30 -0700 Subject: [PATCH 20/34] fix(vmm): stop the removal lock from starving everything that waits on it Holding the per-VM launch lock across removal closed a race and opened two queues behind a wait its own comment measures in hours. A start of a VM being removed asked `removing` only after taking the lock, so it now waited out the whole removal to be told no. It asks before as well -- not instead: the marker can be set while it waits, so the answer under the lock is still the authoritative one. Reconciliation was worse. It takes the lock per VM, in sequence, so one VM stuck in removal stalled the collection of every other VM on the host -- and because the hourly task awaits it, the interval never fires again. The case is real rather than theoretical: an orphaned supervisor process cleaned up by `reload_vms` has no VM in memory and may have no directory, so it is exactly the kind of VM a collection considers dead while its removal is still polling for exit. It takes the lock without waiting now, and skips what it cannot get. That is not a compromise: a held lock means a launch, a stop or a removal of that VM is in flight, and every one of those manages the VM's interfaces itself, so waiting would be waiting for the thing that makes the work unnecessary. --- dstack/vmm/src/app.rs | 82 +++++++++++++++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index d5fe622b3..8546dde18 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -386,15 +386,27 @@ impl App { /// start must not stall unrelated ones. Taken by stop and by removal as /// well as by start: those delete the same interfaces from the other side. pub(crate) async fn launch_lock(&self, id: &str) -> tokio::sync::OwnedMutexGuard<()> { - let lock = { - let mut locks = self.launch_locks.lock().or_panic("mutex poisoned"); - // A VM that is neither starting nor stopping leaves the map holding - // the only reference, so the map stays the size of what is in - // flight rather than of every VM this process has ever touched. - locks.retain(|_, lock| Arc::strong_count(lock) > 1); - locks.entry(id.to_string()).or_default().clone() - }; - lock.lock_owned().await + self.launch_lock_handle(id).lock_owned().await + } + + /// The launch lock, if nothing else holds it. + /// + /// For a caller with something better to do than wait. A held lock means a + /// launch, a stop or a removal of this VM is in flight, and every one of + /// those manages that VM's interfaces itself -- so waiting would be waiting + /// for the very thing that makes the work unnecessary. A removal holds it + /// for as long as the VM takes to exit, which its own comment puts at hours. + pub(crate) fn try_launch_lock(&self, id: &str) -> Option> { + self.launch_lock_handle(id).try_lock_owned().ok() + } + + fn launch_lock_handle(&self, id: &str) -> Arc> { + let mut locks = self.launch_locks.lock().or_panic("mutex poisoned"); + // A VM that is neither starting nor stopping leaves the map holding + // the only reference, so the map stays the size of what is in flight + // rather than of every VM this process has ever touched. + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + locks.entry(id.to_string()).or_default().clone() } pub async fn load_vm( @@ -460,6 +472,14 @@ impl App { Ok(()) } + fn refuse_if_removing(&self, id: &str) -> Result<()> { + let state = self.lock(); + if state.get(id).is_some_and(|vm| vm.state.removing) { + bail!("VM is being removed"); + } + Ok(()) + } + pub async fn start_vm(&self, id: &str) -> Result<()> { self.start_vm_with_restart_policy(id, true).await } @@ -474,17 +494,17 @@ impl App { vm.state.auto_restart.reset(); } } + // Before the lock as well as after it. Removal holds the lock until the + // VM has exited, so a launch that only asked afterwards would wait that + // out -- hours, by removal's own estimate -- to be told no. Asking + // first is not sufficient on its own, because the marker can be set + // while this waits; asking again under the lock is what makes it + // authoritative. + self.refuse_if_removing(id)?; // Everything below reads whether this VM is running and acts on the // answer for as long as the launch takes. See [`App::launch_lock`]. let _launch = self.launch_lock(id).await; - { - let state = self.lock(); - if let Some(vm) = state.get(id) { - if vm.state.removing { - bail!("VM is being removed"); - } - } - } + self.refuse_if_removing(id)?; // A restart decided before a stop must not outlive it. The decision // read `started` from disk; `stop_vm` writes it false under this lock, // so re-reading it here is what makes the stop stick. An explicit start @@ -1104,8 +1124,15 @@ impl App { } for vm_id in dead { // The lock a launch of this VM holds from before it asks netd for - // an interface until after it has recorded one. - let _launch = self.launch_lock(&vm_id).await; + // an interface until after it has recorded one -- and that a + // removal holds until the VM has exited, which can be hours. Taking + // it without waiting is both safe and necessary: whoever holds it + // is already dealing with this VM's interfaces, and waiting would + // stall the collection of every other VM behind one that is busy. + let Some(_launch) = self.try_launch_lock(&vm_id) else { + debug!(vm_id = %vm_id, "not collecting: this VM is busy"); + continue; + }; // Re-read under it. Between the listing and this line the VM may // have been created and started: its directory exists now, and its // interfaces are the ones a launch just built. @@ -2718,6 +2745,23 @@ mod tests { ); } + /// A removal holds the launch lock until the VM has exited, which its own + /// comment puts at hours. Nothing that has something better to do than + /// wait may queue behind it. + #[tokio::test] + async fn work_that_can_wait_does_not_queue_behind_a_removal() { + let app = test_app(); + let removal = app.launch_lock("vm-1").await; + + // The collection skips a busy VM rather than stalling every other VM + // behind it: whoever holds the lock is already dealing with this one. + assert!(app.try_launch_lock("vm-1").is_none()); + assert!(app.try_launch_lock("vm-2").is_some()); + + drop(removal); + assert!(app.try_launch_lock("vm-1").is_some()); + } + /// A restart decided before a stop must not outlive it. The restart task /// reads the started flag off disk and only then queues a launch, which /// waits for the lock the stop is holding; without a re-read under that From f928c1a04e0fb7a31d6469f884addc36b9532443 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 04:35:40 -0700 Subject: [PATCH 21/34] fix(vmm): refuse a stop or an update of a VM being removed, before waiting on it Both took the launch lock that removal holds until the VM has exited, and neither asked whether the VM was being removed at all -- so a stop or an update issued during a removal hung for as long as the removal took, to do work the removal was already doing. Neither has an internal caller; both are RPCs, and an RPC that hangs for hours is worse than one that says why. --- dstack/vmm/src/app.rs | 12 +++++++++++- dstack/vmm/src/main_service.rs | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 8546dde18..ec788ff46 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -472,7 +472,13 @@ impl App { Ok(()) } - fn refuse_if_removing(&self, id: &str) -> Result<()> { + /// Refuses an operation on a VM that is being removed. + /// + /// Cheap, and taken before the launch lock as well as under it. Removal + /// holds that lock until the VM has exited -- hours, by its own estimate -- + /// so anything that only asked afterwards would wait the removal out in + /// order to be told no. + pub(crate) fn refuse_if_removing(&self, id: &str) -> Result<()> { let state = self.lock(); if state.get(id).is_some_and(|vm| vm.state.removing) { bail!("VM is being removed"); @@ -621,6 +627,10 @@ impl App { } pub async fn stop_vm(&self, id: &str) -> Result<()> { + // Removal stops the VM itself and holds the launch lock while it does, + // so this would otherwise wait hours to do again what is already being + // done. + self.refuse_if_removing(id)?; if let Some(vm) = self.lock().get_mut(id) { vm.state.auto_restart.reset(); } diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 7ec47e17d..944e45870 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -1068,6 +1068,9 @@ impl VmmRpc for RpcHandler { let networks = networks_from_proto(&request.networks, &cvm)?; resolve_requested_networks(&networks, &cvm, manifest.vcpu)? }; + // A VM being removed is not one to reconfigure, and removal holds + // this lock until it has exited. + self.app.refuse_if_removing(&request.id)?; // The lock first, then the question. Reading "not running" outside // it and acting on the answer inside is the exact race the lock // exists to close: a launch can start, prepare its interfaces and From 9710e26b66838d9382acf71e5fd3e122b6fdb491 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 04:41:56 -0700 Subject: [PATCH 22/34] fix(vmm): put the CLI help back on the subcommand it describes `netd remove-interface` was inserted between `remove-vm`'s doc comment and its variant, so clap printed "Delete every interface netd holds for one VM" as the help for removing a single interface, and nothing at all for removing a VM's. Three claims that stopped being true, while their code changed underneath them: `Request::Remove`'s `filtered` field is still sent and still required on decode, but the handler no longer derives strictness from it; removal's wait is a SIGKILL teardown, not the "2+ hours" a graceful stop used to take; and what a truncated sweep leaves behind is collected by reconciliation when the caller was the removal, since there is no next launch to do it. --- dstack/vmm/src/app.rs | 10 +++++++--- dstack/vmm/src/main.rs | 10 +++++----- dstack/vmm/src/netd.rs | 9 +++++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index ec788ff46..8de64c212 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1210,7 +1210,8 @@ impl App { warn!( vm_id, "netd stopped releasing this VM's interfaces on its deadline; the rest \ - are released by its next launch or its removal" + go when this VM next launches, or -- once it is removed and nothing \ + claims it -- when reconciliation next runs" ); } } @@ -1355,8 +1356,11 @@ impl App { debug!("graceful VM stop during removal failed: {err:?}"); } - // Poll until the process is no longer running, then remove it. - // Some VMs take a long time to stop (e.g. 2+ hours), so we wait indefinitely. + // Poll until the process is no longer running, then remove it. The + // stop above is a SIGKILL, so this is however long the kernel takes to + // tear the VM down -- seconds for a large TD, unbounded for one wedged + // in a device reset. Waiting is still right: what follows deletes the + // interfaces and the workdir it is using. let mut poll_count: u64 = 0; loop { match self.supervisor.info(id).await { diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 5af0cc5de..627c2fc18 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -94,11 +94,6 @@ enum NetdCommand { #[arg(long)] instance: Option, }, - /// Delete every interface netd holds for one VM. - /// - /// For a VM whose VMM will never ask again -- one whose directory was - /// deleted by hand, or whose instance is gone. A running VMM collects - /// these itself; this is for when there is no longer one to do it. /// Delete one interface by name. /// /// For what nothing else can reach: an interface built before netd @@ -110,6 +105,11 @@ enum NetdCommand { /// The interface name, as `netd list` prints it. name: String, }, + /// Delete every interface netd holds for one VM. + /// + /// For a VM whose VMM will never ask again -- one whose directory was + /// deleted by hand, or whose instance is gone. A running VMM collects + /// these itself; this is for when there is no longer one to do it. RemoveVm { /// The `cvm.instance_id` of the VMM that created them. `netd list` /// shows it. diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 71d7bfa9d..4517e534c 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -301,8 +301,13 @@ pub enum Request { #[serde(flatten)] identity: InterfaceIdentity, /// Whether this interface was created with an nwfilter binding. - /// Macvtap TAPs never carry one, and removal detects them rather than - /// trusting this field. + /// + /// Advisory, and no longer read: removal detects macvtap itself, and + /// clears a binding best-effort whatever this says. The strict rule it + /// used to select exists for prepare, where a binding left at the name + /// blocks the one about to be created; at removal it only left the + /// interface up on the bridge. Still sent, and still required on + /// decode, so a netd that predates that reasoning keeps working. filtered: bool, }, /// Delete every interface netd holds for one VM. From d8d43de21f0d7ee08328614720c5d00131d56ee4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 07:23:22 -0700 Subject: [PATCH 23/34] fix(vmm): keep the removal mark where the removal can be seen `refuse_if_removing` read a flag on the VM's entry, which the orphan cleanup never sets: `spawn_finish_remove` marks nothing when the ID has no entry, and that is exactly the case it exists for. So `finish_remove_vm` held the launch lock across a whole teardown while every operation on that ID was told it was free to proceed -- and `StartVm` then waited the teardown out with no error and no log, only to fail at the end on a VM no longer in memory, after it had already written started to disk. The mark now lives in the state, not in the entry, so an ID with no entry can carry one. It is cleared by a guard rather than by the last step of the happy path: `finish_remove_vm` has two `?`s after the wait, and a mark left behind by either is not a stale flag but a VM no operation can reach again, including the removal that would retry. `claimable_vm_ids` hands back the set it built instead of collecting it into a vector for the caller to linear-scan once per interface. --- dstack/vmm/src/app.rs | 113 +++++++++++++++++++++++++++++++++++------- 1 file changed, 96 insertions(+), 17 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 8de64c212..578bda987 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -362,6 +362,7 @@ impl App { state: Arc::new(Mutex::new(AppState { cid_pool, vms: HashMap::new(), + removing: HashSet::new(), })), config: Arc::new(config), pull_status: Arc::new(Mutex::new(std::collections::HashMap::new())), @@ -479,8 +480,7 @@ impl App { /// so anything that only asked afterwards would wait the removal out in /// order to be told no. pub(crate) fn refuse_if_removing(&self, id: &str) -> Result<()> { - let state = self.lock(); - if state.get(id).is_some_and(|vm| vm.state.removing) { + if self.lock().is_removing(id) { bail!("VM is being removed"); } Ok(()) @@ -995,7 +995,7 @@ impl App { /// `None` when the answer cannot be established, which is not the same as /// nobody claiming anything: an unreadable VM directory read as empty /// would offer every interface on the host up for collection. - fn claimable_vm_ids(&self) -> Option> { + fn claimable_vm_ids(&self) -> Option> { let mut ids: HashSet = self.lock().vms.keys().cloned().collect(); match fs::read_dir(self.vm_dir()) { Ok(entries) => { @@ -1051,7 +1051,7 @@ impl App { return None; } } - Some(ids.into_iter().collect()) + Some(ids) } /// Deletes every host interface netd holds for a VM this instance no @@ -1311,12 +1311,11 @@ impl App { pub async fn remove_vm(&self, id: &str) -> Result<()> { { let mut state = self.lock(); - let vm = state.get_mut(id).context("VM not found")?; - if vm.state.removing { + state.get(id).context("VM not found")?; + if !state.start_removing(id) { // Already being removed — idempotent return Ok(()); } - vm.state.removing = true; } // Persist the removing marker so crash recovery can resume @@ -1342,6 +1341,12 @@ impl App { /// /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup. async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> { + // Every exit from here clears the mark, including the `?`s below and a + // panic in this task, which `tokio::spawn` would otherwise swallow. + let _mark = RemovalMark { + app: self.clone(), + id: id.to_string(), + }; // Held across the stop, the wait and the release, not just the release. // `removing` turns launches away, but a launch that passed that check // before the marker was set is already inside the lock: it has not @@ -1428,16 +1433,13 @@ impl App { /// Returns false if a cleanup task is already running for this VM. fn spawn_finish_remove(&self, id: &str) -> bool { { - let mut state = self.lock(); - if let Some(vm) = state.get_mut(id) { - if vm.state.removing { - // Already being cleaned up — skip - return false; - } - vm.state.removing = true; + // Claimed in the set rather than in the entry: an orphaned + // supervisor process has no entry, and that is exactly the case + // where the launch lock is held with nothing turning waiters away. + if !self.lock().start_removing(id) { + // Already being cleaned up — skip + return false; } - // If VM is not in memory (e.g. orphaned supervisor process), no entry to guard - // but we still need to clean up the supervisor process. } let app = self.clone(); let id = id.to_string(); @@ -2554,7 +2556,7 @@ mod tests { ); assert_eq!( app.claimable_vm_ids(), - Some(vec!["vm-that-did-not-load".to_string()]) + Some(HashSet::from(["vm-that-did-not-load".to_string()])) ); // Unreadable is not "nobody claims anything", which would offer every @@ -2776,6 +2778,43 @@ mod tests { assert!(app.try_launch_lock("vm-1").is_some()); } + /// The orphan cleanup runs for an ID that never loaded, so a guard living + /// in the VM entry is not there when it holds the launch lock across the + /// whole teardown. Without the mark, `StartVm` on that ID waits the + /// teardown out with no error and no log. + #[tokio::test] + async fn a_removal_with_no_vm_entry_still_turns_operations_away() { + let app = test_app(); + assert!(app.refuse_if_removing("orphan").is_ok()); + assert!(app.lock().start_removing("orphan")); + assert!( + app.refuse_if_removing("orphan").is_err(), + "an orphan being removed is still a VM being removed" + ); + // And the same removal cannot be started twice. + assert!(!app.lock().start_removing("orphan")); + } + + /// `finish_remove_vm` returns early on more than its happy path. A mark it + /// left behind is not a stale flag: every later operation on that VM, + /// including the removal that would retry, answers "being removed". + #[tokio::test] + async fn a_removal_that_gives_up_early_does_not_leave_the_vm_marked() { + let app = test_app(); + { + let _mark = RemovalMark { + app: app.clone(), + id: "vm-1".to_string(), + }; + assert!(app.lock().start_removing("vm-1")); + assert!(app.refuse_if_removing("vm-1").is_err()); + } + assert!( + app.refuse_if_removing("vm-1").is_ok(), + "the mark is cleared however the removal ends" + ); + } + /// A restart decided before a stop must not outlive it. The restart task /// reads the started flag off disk and only then queues a launch, which /// waits for the lock the stop is holding; without a re-read under that @@ -3933,6 +3972,14 @@ impl VmState { pub(crate) struct AppState { cid_pool: IdPool, vms: HashMap, + /// The VMs a removal is currently working on. + /// + /// Separate from `VmState::removing` because the set has to outlive the + /// entry. Orphan cleanup runs for IDs that never loaded into `vms`, and + /// `finish_remove_vm` holds the launch lock across the whole teardown, so + /// a guard that lives in the entry cannot turn away the operation that + /// would otherwise wait that teardown out. + removing: HashSet, } impl AppState { @@ -3955,6 +4002,38 @@ impl AppState { pub fn iter_vms(&self) -> impl Iterator { self.vms.values() } + + /// Claims `id` for a removal. False when one already has it. + fn start_removing(&mut self, id: &str) -> bool { + if let Some(vm) = self.vms.get_mut(id) { + vm.state.removing = true; + } + self.removing.insert(id.to_string()) + } + + fn is_removing(&self, id: &str) -> bool { + self.removing.contains(id) + } +} + +/// Clears the in-flight removal mark however the removal ends. +/// +/// `finish_remove_vm` returns early on more than its happy path, and a mark +/// left behind is not a stale flag: every operation on that VM answers "being +/// removed" from then on, including the removal that would retry. +struct RemovalMark { + app: App, + id: String, +} + +impl Drop for RemovalMark { + fn drop(&mut self) { + let mut state = self.app.lock(); + state.removing.remove(&self.id); + if let Some(vm) = state.vms.get_mut(&self.id) { + vm.state.removing = false; + } + } } /// Reject VM ids that would escape `run_path` once joined into a filesystem From 69c34f08c91c2d11fc6e2c9dc2c91ac1f0e85500 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 07:23:22 -0700 Subject: [PATCH 24/34] fix(vmm): hold an update to one VM against the removal of that VM `update_vm` asked whether the VM was being removed only inside the branch that changes networking, and dropped the launch lock at the end of that branch. What follows is the part that writes: `put_manifest` creates the directory it writes into. An update that resumed after a removal deleted that directory recreated it holding nothing but a manifest -- invisible to `list_vms`, failing to load at every start, answering "VM not found" to a second removal, and claiming that VM's netd interfaces against collection for as long as the VMM runs. The interfaces a netd outage kept the removal from releasing then had nothing left that could reach them. The refusal moves to the top, before the compose file is written, and the lock is taken there and held to the end, with the refusal repeated under it. `validate_port_mapping_nics` is gated the way the publishability check beside it already is. It ran on every update, against the node's current default, so changing `cvm.networking.nic.mode` made every later update of a VM that pinned a mapping fail over a field the request never touched -- the unmanageable- rather-than-fixed outcome the comment three lines below it rules out. --- dstack/vmm/src/main_service.rs | 53 ++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 944e45870..98295ca9b 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -988,6 +988,21 @@ impl VmmRpc for RpcHandler { async fn update_vm(self, request: UpdateVmRequest) -> Result { info!(vm_id = %request.id, "update_vm RPC called"); + // A VM being removed is not one to reconfigure. Before the lock, + // because removal holds it across the whole teardown and anything that + // only asked afterwards would wait that out in order to be told no. + self.app.refuse_if_removing(&request.id)?; + // Held from here rather than around the parts that touch the host, + // because everything below writes into the workdir -- the compose + // file first, the manifest last -- and `put_manifest` creates the + // directory it writes into. An update that resumed after a removal + // deleted that directory would recreate it holding nothing but a + // manifest: invisible to `list_vms`, unloadable at every start, and + // claiming the VM's netd interfaces against collection forever. + let _launch = self.app.launch_lock(&request.id).await; + // Again under the lock: removal can have claimed the VM while this + // waited for it. + self.app.refuse_if_removing(&request.id)?; let new_id = if !request.compose_file.is_empty() { // check the compose file is valid let _app_compose: AppCompose = @@ -1068,16 +1083,12 @@ impl VmmRpc for RpcHandler { let networks = networks_from_proto(&request.networks, &cvm)?; resolve_requested_networks(&networks, &cvm, manifest.vcpu)? }; - // A VM being removed is not one to reconfigure, and removal holds - // this lock until it has exited. - self.app.refuse_if_removing(&request.id)?; - // The lock first, then the question. Reading "not running" outside - // it and acting on the answer inside is the exact race the lock - // exists to close: a launch can start, prepare its interfaces and - // deploy QEMU in between, and the release would then delete the - // interfaces of a VM that is running -- silently, since QEMU stays - // up and the supervisor still reports it healthy. - let _launch = self.app.launch_lock(&request.id).await; + // Under the launch lock this whole call holds. Reading "not + // running" outside it and acting on the answer inside is the exact + // race the lock exists to close: a launch can start, prepare its + // interfaces and deploy QEMU in between, and the release would + // then delete the interfaces of a VM that is running -- silently, + // since QEMU stays up and the supervisor still reports it healthy. let is_running = self .app .supervisor @@ -1093,18 +1104,18 @@ impl VmmRpc for RpcHandler { } manifest.networks = networks; } - // After both, since either half can move and the other still has to - // agree with it. - validate_port_mapping_nics( - &manifest.port_map, - &resolved_nic_modes(&manifest.networks, &self.app.config.cvm, manifest.vcpu), - )?; - // Only when this request moved one of them. A VM deployed before the - // node could answer for its ports must stay editable in every other - // respect: read-modify-write sends the whole configuration back, and - // refusing a memory change over a port mapping nobody touched would - // make the VM unmanageable rather than fixed. + // Both only when this request moved one of the two halves, and after + // both, since either half can move and the other still has to agree + // with it. A VM deployed before the node could answer for its ports + // must stay editable in every other respect: read-modify-write sends + // the whole configuration back, and refusing a memory change over a + // port mapping nobody touched -- or over a node default that changed + // under it -- would make the VM unmanageable rather than fixed. if request.update_ports || request.update_networking { + validate_port_mapping_nics( + &manifest.port_map, + &resolved_nic_modes(&manifest.networks, &self.app.config.cvm, manifest.vcpu), + )?; self.refuse_unpublishable_ports(&manifest).await?; } let compose_file = fs::read_to_string(vm_work_dir.app_compose_path()) From d70c8d2d33111f94d9a833dea63e4298248db74c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 07:23:37 -0700 Subject: [PATCH 25/34] fix(vmm): stop a netd the VMM cannot open from reading as one that is not there Every `connect` failure carried the `Unreachable` marker, and both callers that treat absence as "nothing to do on this host" -- the interface release and the periodic collection -- skip their work at `debug!` when they see it. The default configuration reaches that state: netd runs as root and chmods its socket to `0660`, while the VMM is meant to run unprivileged, so a VMM whose user is not in root's group gets `EACCES` on every call. Interfaces then accumulate with nothing said at the default filter, while `create_vm` tells the operator to go and run a netd that is already running. Only the two errnos that mean nothing is listening are read as absence now. `hello` is answered before the blocking pool rather than inside it. The pool is finite and its tasks cannot be cancelled, so a node whose `virsh` calls are all timing out fills it, and a `hello` queued behind them times out too -- putting back the busy-netd-reads-as-absent this daemon exists to avoid. A sweep that deleted an interface but could not clear its nwfilter binding reported success, while the orphaned-binding half of the same loop treated the same failure as an error. It reports the pass as incomplete, which is what the stop it came from already knows how to say. `owner_of` strips what sysfs appended rather than trimming the whole alias: an identity with trailing whitespace re-derived a name that is not the one it is on, leaving the interface permanently unattributable -- never collected, and removable only by hand. --- dstack/vmm/src/netd.rs | 56 +++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 4517e534c..5044696b1 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -509,7 +509,14 @@ pub fn interface_alias(identity: &InterfaceIdentity) -> String { /// `vm_id` containing the separator still parses. An `instance_id` containing /// one does not, and is refused at prepare rather than mis-parsed here. pub fn owner_of(tap: &str, alias: &str) -> Option { - let rest = alias.trim().strip_prefix(ALIAS_PREFIX)?.strip_prefix(':')?; + // `trim_end_matches`, not `trim`: sysfs adds a newline, and a `vm_id` + // whose own trailing whitespace were trimmed off here would re-derive a + // name that is not the one it is on, making the interface permanently + // unattributable -- never collected, only removable by hand. + let rest = alias + .trim_end_matches(['\n', '\r']) + .strip_prefix(ALIAS_PREFIX)? + .strip_prefix(':')?; let (nic_index, rest) = rest.split_once(':')?; let (instance_id, vm_id) = rest.split_once(':')?; let identity = InterfaceIdentity { @@ -755,11 +762,26 @@ async fn exchange(socket: &Path, request: &Request) -> Result { Request::Check { .. } => "check", }; let exchange = async { - let mut stream = UnixStream::connect(socket) - .await - .map_err(anyhow::Error::from) - .context(Unreachable) - .with_context(|| format!("failed to connect to netd at {}", socket.display()))?; + let mut stream = UnixStream::connect(socket).await.map_err(|error| { + // Only the two errnos that mean "nothing is listening". A socket + // the VMM's user cannot open (`EACCES`, the default `0660` on a + // root-owned socket) or a VMM out of descriptors would otherwise + // read as an absent netd, and every caller that treats absence as + // "nothing to do here" would skip its work at `debug!` -- leaking + // interfaces on a host whose netd is running fine, while telling + // the operator to go start one. + let absent = matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::ConnectionRefused + ); + let error = anyhow::Error::from(error) + .context(format!("failed to connect to netd at {}", socket.display())); + if absent { + error.context(Unreachable) + } else { + error + } + })?; let message = serde_json::to_vec(request)?; if message.len() as u64 > MAX_MESSAGE_SIZE { bail!("netd request is too large"); @@ -885,6 +907,12 @@ async fn serve_connection( debug!("netd liveness probe"); return Ok(()); } + // Answered here rather than in `handle_request`, which reaches the + // blocking pool first. That pool is finite and its tasks cannot be + // cancelled, so a node whose `virsh` calls are all timing out fills it + // -- and `hello` queued behind them times out too, putting back the + // "a busy netd reads as an absent one" this daemon exists to avoid. + Ok(Some(Request::Hello)) => Ok(Outcome::Hello(capabilities())), Ok(Some(request)) => { // `handle_request` shells out to `ip` and `virsh` and waits on the // operation lock, so it can block for as long as those take. On a @@ -1225,14 +1253,21 @@ enum BindingCleanup { /// after it has failed is how a hung `libvirtd` turns a bounded collection into /// an unbounded one: the deadline is reached having deleted nothing, while /// every other VM on the host waits behind the operation lock. -fn remove_interface_in_pass(uri: &str, tap: &str, libvirt: &mut bool) -> Result<()> { +/// `Ok(false)` when the interface is gone but its binding could not be +/// cleared. The pass reports that as incomplete rather than as success: the +/// orphan half of the same loop already treats a binding it could not delete +/// as a failure, and a stale binding is exactly the state a later filtered +/// prepare at the same name hard-fails on. +fn remove_interface_in_pass(uri: &str, tap: &str, libvirt: &mut bool) -> Result { + let mut cleared = true; if *libvirt && !is_macvtap(tap) { if let Err(error) = delete_binding(uri, tap) { warn!(%tap, "could not clear a possible nwfilter binding: {error:#}"); *libvirt = false; + cleared = false; } } - remove_interface(uri, tap, BindingCleanup::Skip) + remove_interface(uri, tap, BindingCleanup::Skip).map(|()| cleared) } /// Deletes every interface a VM could hold, by deriving each name rather than @@ -1303,9 +1338,12 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res warn!(%tap, %error, "failed to remove interface"); first_error.get_or_insert(error); } - Ok(()) => { + Ok(cleared) => { info!(%tap, %vm_id, "removed interface"); removed += 1; + if !cleared { + incomplete = true; + } } } } From 1c75cf38687804bbe89385460bafb814e5caee52 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 07:23:37 -0700 Subject: [PATCH 26/34] fix(vmm/ui): carry the NIC a port mapping was pinned to `normalizePorts` built the pin and then rebuilt the object without it in a trailing `map`, so the web UI could not pin a mapping at all -- and an edit made in the UI silently unpinned a mapping that had been pinned from the CLI, which the composable's own comment says cannot happen. TypeScript did not catch it because the field is optional. Three more, all in the same feature: - A cleared NIC box pinned NIC 0. `v-model.number` hands back the raw string when it does not parse, and `Number('')` is 0. - The update dialog never passed `nic-count`, so its NIC column was always hidden. A VM shrunk to one NIC could not have a stale pin cleared. - The update always sent `update_ports`, so the server-side "only when this request moved one of them" gate did not cover the UI at all: on a node whose netd does not forward host ports, a VM with any port mapping could not be edited from the UI in any respect. It is sent only when the mappings differ from what the dialog opened with. --- .../ui/src/components/PortMappingEditor.ts | 2 +- .../vmm/ui/src/components/UpdateVmDialog.ts | 5 ++- dstack/vmm/ui/src/composables/useVmManager.ts | 34 +++++++++++++------ 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/dstack/vmm/ui/src/components/PortMappingEditor.ts b/dstack/vmm/ui/src/components/PortMappingEditor.ts index 88f0c945c..70a4ad278 100644 --- a/dstack/vmm/ui/src/components/PortMappingEditor.ts +++ b/dstack/vmm/ui/src/components/PortMappingEditor.ts @@ -11,7 +11,7 @@ type PortEntry = { // Which NIC the traffic enters through. Blank lets the VMM pick: the first // user-mode NIC, else the first bridge NIC. A single-NIC VM never needs it, // which is why the field only appears once a VM has more than one. - nic_index?: number | null; + nic_index?: number | string | null; }; // ... keep your types as-is ... diff --git a/dstack/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts index ec2bfbfa6..4fa40e493 100644 --- a/dstack/vmm/ui/src/components/UpdateVmDialog.ts +++ b/dstack/vmm/ui/src/components/UpdateVmDialog.ts @@ -176,7 +176,10 @@ const UpdateVmDialogComponent = {
- +
diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index 79ed62984..7b0bd846c 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -113,7 +113,9 @@ type PortFormEntry = { * the whole list back, so dropping it here would silently unpin a mapping * whenever anyone touched an unrelated field. */ - nic_index?: number | null; + // `v-model.number` leaves the raw string here when it does not parse, so + // an emptied box is `''` rather than `null`. See `normalizePorts`. + nic_index?: number | string | null; }; type NetworkFormEntry = { @@ -187,6 +189,9 @@ type UpdateDialogState = { disk_size: number; image: string; ports: PortFormEntry[]; + /// What the dialog opened with, normalized, so the update can tell whether + /// this request moved the port mappings at all. + originalPorts: VmmTypes.IPortMapping[]; attachAllGpus: boolean; selectedGpus: string[]; updateGpuConfig: boolean; @@ -278,6 +283,7 @@ function createUpdateDialogState(): UpdateDialogState { disk_size: 0, image: '', ports: [], + originalPorts: [], attachAllGpus: false, selectedGpus: [], updateGpuConfig: false, @@ -455,8 +461,11 @@ fi port.vm_port === null || port.vm_port === undefined ? Number.NaN : Number(port.vm_port); // An unpinned mapping must stay unpinned rather than become NIC 0: // the VMM's own default is the first user-mode NIC, not the first NIC. + // `v-model.number` hands back the raw string when it does not parse, + // so a box the operator cleared arrives as `''`. `Number('')` is 0, + // which would pin to NIC 0 the mapping they just unpinned. const nicIndex = - port.nic_index === null || port.nic_index === undefined + port.nic_index === null || port.nic_index === undefined || port.nic_index === '' ? undefined : Number(port.nic_index); return { @@ -474,13 +483,7 @@ fi port.protocol.length > 0 && Number.isFinite(port.host_port) && Number.isFinite(port.vm_port), - ) - .map((port) => ({ - protocol: port.protocol, - host_address: port.host_address, - host_port: port.host_port, - vm_port: port.vm_port, - })); + ); const cloneNetworks = (configuration?: VmConfiguration | null): NetworkFormEntry[] => { const configured = configuration?.networks && configuration.networks.length > 0 @@ -1145,6 +1148,10 @@ type CreateVmPayloadSource = { disk_size: config.disk_size || 0, image: config.image || '', ports: clonePortMappings(config.ports || []), + // What the dialog opened with. `update_ports` means "this request moved + // the port mappings", and the server refuses some mappings it has to + // keep accepting on a request that only touched memory. + originalPorts: normalizePorts(clonePortMappings(config.ports || [])), attachAllGpus: gpuSelection.attachAll, selectedGpus: gpuSelection.selected, updateGpuConfig: false, @@ -1324,8 +1331,13 @@ type CreateVmPayloadSource = { body.compose_file = composeNeedsUpdate ? await makeUpdateComposeFile() : undefined; body.encrypted_env = encryptedEnvPayload; body.user_config = updated.user_config; - body.update_ports = true; - body.ports = normalizePorts(updated.ports); + const ports = normalizePorts(updated.ports); + const portsMoved = + JSON.stringify(ports) !== JSON.stringify(updateDialog.value.originalPorts ?? []); + if (portsMoved) { + body.update_ports = true; + body.ports = ports; + } body.gpus = updateDialog.value.updateGpuConfig ? configGpu(updated, true) : undefined; if (updated.updateNetworking) { body.update_networking = true; From e04b11a69a7e2a69a74cc486f03441c35983a706 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 07:23:45 -0700 Subject: [PATCH 27/34] docs: stop recommending the setuid helper this PR stopped using `setup-bridge.sh check` still failed the node when `qemu-bridge-helper` was not setuid root and `/etc/qemu/bridge.conf` had no `allow` line, three pages after the doc it is recommended by says neither is needed any more. Following both left a setuid binary nothing uses and a standing grant for any local user to attach a TAP to the bridge. The checks and the setup steps go; the teardown still removes the `allow` line, now saying why. Four statements corrected to match the code: - `dstack-vmm netd -c vmm.toml` does not parse. `--config` is not a global argument, so it has to come before the subcommand, as every other doc has it. - A VM on user networking does contact the netd socket: the release runs on every launch and every stop, before the decision about whether anything needs building. Nothing about the VM depends on the answer, which is the part worth saying. - The `instance_id` comment described two VMMs collecting each other's running VMs as a live hazard. A VMM that finds another live instance on its value refuses to start. - The CLI guide's port mapping section never mentioned `@`, which is in `--help` and in the onboarding doc. --- docs/bridge-networking.md | 2 +- docs/libvirt-network-filter.md | 6 +- docs/vmm-cli-user-guide.md | 5 ++ dstack/scripts/setup-bridge.sh | 112 +-------------------------------- dstack/vmm/vmm.toml | 4 +- 5 files changed, 16 insertions(+), 113 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index d70d1c96d..a3e181530 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -149,7 +149,7 @@ Bridge networking needs `netd`, the privileged helper that owns every host interface a bridge or macvtap NIC uses. It is the same binary: ```bash -sudo dstack-vmm netd -c vmm.toml +sudo dstack-vmm --config vmm.toml netd ``` Nothing else on the node needs `CAP_NET_ADMIN`: the VMM itself still runs diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index 7bad88ba7..3a1313aef 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -219,8 +219,10 @@ sudo dstack-vmm --config ./vmm.toml \ ``` User networking and a caller-supplied netdev never ask `netd` to build an -interface, and a VM using only those never contacts it. Bridge and macvtap -always do, and fail closed if `netd` is unavailable. +interface. The VMM still contacts the socket for such a VM -- every launch and +every stop releases whatever the VM held, before it decides whether it needs +anything built -- but nothing about the VM depends on the answer. Bridge and +macvtap do ask, and fail closed if `netd` is unavailable. Filtered TAP netdevs follow the node's `vhost` and `queues` settings like any other TAP-backed NIC (see [network-data-plane.md](network-data-plane.md)). The diff --git a/docs/vmm-cli-user-guide.md b/docs/vmm-cli-user-guide.md index 6a669ca4e..1f803e31c 100644 --- a/docs/vmm-cli-user-guide.md +++ b/docs/vmm-cli-user-guide.md @@ -290,6 +290,11 @@ Expose services running in your VM: # Multiple ports --port tcp:8080:80 --port tcp:8443:443 + +# Pin a mapping to one NIC: protocol[:host_address]:host_port:vm_port@ +# Without @ the mapping enters through the first user-mode NIC, or the +# first bridge NIC when the VM has no user-mode one. +--port tcp:0.0.0.0:8443:443@0 ``` #### GPU Assignment diff --git a/dstack/scripts/setup-bridge.sh b/dstack/scripts/setup-bridge.sh index edcbe35a7..ecc3d2901 100755 --- a/dstack/scripts/setup-bridge.sh +++ b/dstack/scripts/setup-bridge.sh @@ -44,83 +44,6 @@ run_cmd() { fi } -# --- Detect qemu-bridge-helper path --- - -find_bridge_helper() { - local paths=( - /usr/lib/qemu/qemu-bridge-helper - /usr/libexec/qemu-bridge-helper - /usr/local/lib/qemu/qemu-bridge-helper - /usr/local/libexec/qemu-bridge-helper - ) - for p in "${paths[@]}"; do - if [[ -f "$p" ]]; then - echo "$p" - return 0 - fi - done - return 1 -} - -# --- Detect current bridge provider --- - -# Returns "libvirt:" if bridge is managed by a libvirt network, -# "standalone" otherwise. -detect_bridge_provider() { - if command -v virsh &>/dev/null; then - local name br - while read -r name; do - [[ -z "$name" ]] && continue - br=$(virsh net-dumpxml "$name" 2>/dev/null | grep -oP "/dev/null) - fi - echo "standalone" -} - -# --- Check functions --- - -check_bridge_helper() { - echo - bold "qemu-bridge-helper" - local helper - if ! helper=$(find_bridge_helper); then - check_fail "qemu-bridge-helper not found" - check_info "Install QEMU: sudo apt install qemu-system-x86" - return - fi - check_pass "found at $helper" - - if [[ -u "$helper" ]]; then - check_pass "setuid bit is set" - else - check_fail "setuid bit not set" - check_info "Fix: sudo chmod u+s $helper" - fi -} - -check_bridge_conf() { - echo - bold "/etc/qemu/bridge.conf" - local conf="/etc/qemu/bridge.conf" - if [[ ! -f "$conf" ]]; then - check_fail "$conf does not exist" - check_info "Fix: sudo mkdir -p /etc/qemu && echo 'allow $BRIDGE' | sudo tee $conf" - return - fi - check_pass "$conf exists" - - if grep -qE "^allow[[:space:]]+($BRIDGE|all)[[:space:]]*$" "$conf" 2>/dev/null; then - check_pass "bridge '$BRIDGE' is allowed" - else - check_fail "bridge '$BRIDGE' not found in $conf" - check_info "Fix: echo 'allow $BRIDGE' | sudo tee -a $conf" - fi -} - check_bridge_interface() { echo bold "bridge interface: $BRIDGE" @@ -326,33 +249,6 @@ check_forward_rules() { # --- Setup: common --- -setup_bridge_conf() { - echo - bold "Setting up /etc/qemu/bridge.conf" - run_cmd sudo mkdir -p /etc/qemu - if [[ -f /etc/qemu/bridge.conf ]] && grep -qE "^allow[[:space:]]+($BRIDGE|all)" /etc/qemu/bridge.conf 2>/dev/null; then - echo " already configured" - else - run_cmd bash -c "echo 'allow $BRIDGE' | sudo tee -a /etc/qemu/bridge.conf" - fi -} - -setup_bridge_helper() { - echo - bold "Setting up qemu-bridge-helper" - local helper - if ! helper=$(find_bridge_helper); then - echo " $(red 'ERROR'): qemu-bridge-helper not found. Install QEMU first." - return 1 - fi - if [[ -u "$helper" ]]; then - echo " setuid already set on $helper" - else - run_cmd sudo chmod u+s "$helper" - echo " setuid set on $helper" - fi -} - setup_ip_forward() { echo bold "Enabling IP forwarding" @@ -681,8 +577,6 @@ cmd_check() { echo "provider: $(bold 'standalone')" fi - check_bridge_helper - check_bridge_conf check_bridge_interface check_dhcp check_dhcp_firewall @@ -728,8 +622,6 @@ cmd_setup() { $DRY_RUN && echo "dry-run: $(yellow 'yes')" # Common setup - setup_bridge_conf - setup_bridge_helper setup_ip_forward # Mode-specific setup @@ -812,7 +704,9 @@ cmd_destroy() { fi fi - # Remove bridge.conf entry + # Remove the bridge.conf entry an older setup added. netd owns every host + # interface now, so qemu-bridge-helper is not used and the `allow` line is + # a standing grant to attach any local user's TAP to the bridge. local conf="/etc/qemu/bridge.conf" if [[ -f "$conf" ]] && grep -qE "^allow[[:space:]]+${BRIDGE}[[:space:]]*$" "$conf" 2>/dev/null; then echo diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index eb3875b14..11354a055 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -49,7 +49,9 @@ qmp_socket = false # empty, a stable value is derived from run_path. It is what netd records on # every host interface this VMM asks for, and what a collection uses to tell # this instance's interfaces from another's -- so two VMMs sharing one value on -# one host would each collect the other's running VMs. May not contain ":". +# one host would each collect the other's running VMs. A VMM that finds another +# live instance using the value it was given refuses to start rather than let +# that happen. May not contain ":". instance_id = "" # Network choices that deployment RPC callers may make. Macvtap is excluded # by default because libvirt nwfilter applies only to bridge interfaces. From 443323a433e711653fd5d3792b536ac0171a3b8e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 07:57:50 -0700 Subject: [PATCH 28/34] refactor(vmm): stop tracking whether the host publishes a port `netd` in this repository builds interfaces; it does not forward host ports. `prepare_bridge` validates the `ingress` field of a request and then never reads it, and `capabilities()` said `ingress: false` unconditionally. Every mechanism built on top of that answer therefore described a forwarding netd that does not exist -- and each one had to be right about a host state the VMM does not control and cannot re-check, which is a bug surface bought with nothing. Gone, and with them 700 lines: - `PortMapping.published` in the RPC, `published_at`, `IngressBinding` and its `answers`, `Networking.ingress`, and the echo-and-compare in the launch path. No shipped client ever surfaced the field. - `refuse_unpublishable_ports`. It made `create_vm` with a port mapping fail outright on every bridge node running this repository's netd -- including `dstackup install`, which creates the KMS VM with one -- and refused on a netd that was merely restarting. What is left is the part that is a fact about the VM rather than about the host: a pin to a NIC that is macvtap or does not exist, or a VM with no user-mode and no bridge NIC at all. That check is local, synchronous, and now covers the unpinned case too, so `create_vm` and `update_vm` ask exactly one question about port mappings. - The `hello` handshake: `Capabilities`, `Reachability`, `probe`, the 30-second answer cache, `supports`, `records_ownership`, `forwards_ingress`. It existed to tell "netd does not have this operation" from "the operation failed", to decide between a sweep and a fallback. With the fallback gone there is nothing to decide: the release asks netd to sweep, and a refusal is a warning. What it holds is reclaimed by the VM's next launch or by reconciliation. - `release_recorded_interfaces` and `LEGACY_TEARDOWN_SPAN`, the eight-round-trip teardown for a netd too old to sweep. netd ships in this binary; the fix for one that cannot sweep is to restart it. `release_vm_interfaces` no longer takes the recorded networks -- it sweeps by identity, which is what made the record unnecessary -- so three call sites stop reading `runtime_networks` to hand it something it ignored, and `finish_remove_vm` loses one of the two `?`s that could strand a removal. --- docs/bridge-networking.md | 28 +- dstack/crates/dstack-cli-core/src/ports.rs | 2 - dstack/crates/dstackup/src/install.rs | 2 - dstack/vmm/rpc/proto/vmm_rpc.proto | 8 - dstack/vmm/src/app.rs | 278 +++------------ dstack/vmm/src/app/network.rs | 1 - dstack/vmm/src/app/vm_info.rs | 114 +------ dstack/vmm/src/config.rs | 9 - dstack/vmm/src/main_service.rs | 90 ++--- dstack/vmm/src/netd.rs | 378 +-------------------- 10 files changed, 91 insertions(+), 819 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index a3e181530..405b713a7 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -226,22 +226,18 @@ A mapping resolves to exactly one NIC, and that NIC's backend decides the mechanism: `hostfwd=` for user mode, `netd` for a bridge. Nothing can be claimed by both. -### Which ports are actually published - -A port mapping is a *request*. Whether it is met depends on which NIC carries -it: QEMU publishes a user-mode NIC's mappings itself, while a bridge NIC's can -only be published by `netd`, and the `netd` in this repository builds -interfaces and does not forward host ports. - -`GetInfo` reports `published` per mapping so the difference is visible rather -than assumed. A deployment that asks for something this node cannot publish is -refused outright — nothing is running on the answer yet — while a VM deployed -before the node could answer only gets a warning at launch, so an upgrade never -turns a silent misconfiguration into an outage. - -To publish a bridge NIC's ports, run a `netd` that forwards. It reports -`ingress: true` in its `hello` and echoes what it established in each prepare; -the VMM records that answer and holds it to it per mapping. +### Which ports a bridge NIC can publish + +QEMU publishes a user-mode NIC's mappings itself, with `hostfwd=`. A bridge +NIC's would have to be published by `netd`, and **the `netd` in this repository +builds interfaces; it does not forward host ports.** A mapping that resolves to +a bridge NIC is carried to `netd` in the prepare and goes no further. + +The VMM does not track whether the host is forwarding. It reports what a VM +asked for, and a mapping is refused only when the VM's own topology gives it +nowhere to go — a pinned NIC that is macvtap or does not exist, or a VM with no +user-mode and no bridge NIC at all. Those are facts about the VM, decided +without asking the host anything. ## Who owns an interface diff --git a/dstack/crates/dstack-cli-core/src/ports.rs b/dstack/crates/dstack-cli-core/src/ports.rs index 03246c27b..86bca1c9c 100644 --- a/dstack/crates/dstack-cli-core/src/ports.rs +++ b/dstack/crates/dstack-cli-core/src/ports.rs @@ -78,8 +78,6 @@ pub fn parse_port(spec: &str) -> Result { vm_port, nic_index, // A request, not an answer: the server reports what it actually - // published back through `GetInfo`. - published: None, }) } diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index 41ee0f9d2..20852f749 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -298,8 +298,6 @@ pub(crate) async fn cmd_install(mut o: InstallOpts, release_api_base_url: &str) // is one NIC, and the VMM resolves that itself. nic_index: None, // A request, not an answer: the VMM reports what it - // actually published back through `GetInfo`. - published: None, }], ..Default::default() }; diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 72fa2fb87..239ac2d79 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -191,14 +191,6 @@ message PortMapping { // user-mode NIC for management -- the topology multi-NIC was added for -- // put every published port on the management NIC. optional uint32 nic_index = 5; - // Whether the host port is actually reachable at the guest right now. - // - // Absent for a VM that is not running, and from a VMM that predates the - // field. A mapping was only ever a request: on a bridge NIC it is met by the - // node's netd, which may not forward host ports at all, and reporting the - // request as though it were the answer is how published ports could be - // listed for a VM nothing on the host forwarded any traffic to. - optional bool published = 6; } // Partial configuration used when mutating an existing VM. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 578bda987..28715f3bb 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -316,27 +316,8 @@ pub struct App { /// One lock per VM, held across a launch or a teardown. See /// [`App::launch_lock`]. launch_locks: Arc>>>>, - /// What this node's netd last said it can do, and when. See - /// [`App::netd_capabilities`]. - netd_probe: Arc>>, } -/// How long a netd capability answer is reused. Short, because netd is -/// upgraded and restarted under a running VMM, and a VMM that cached "this one -/// cannot sweep" across the upgrade that gave it the ability would keep -/// falling back for as long as it stayed up. -const NETD_PROBE_TTL: Duration = Duration::from_secs(30); - -/// How far past a VM's recorded NIC count a teardown reaches when netd is too -/// old to sweep by identity. -/// -/// The record is what a legacy netd leaves us: it cannot derive the space -/// itself, and asking it for all 256 possible indices would be 256 round trips -/// on every stop. Eight covers a lost NIC or two on any topology anyone -/// deploys, each miss is one cheap no-op, and what it does not reach is -/// collected the next time the node runs a netd that can sweep. -const LEGACY_TEARDOWN_SPAN: usize = 8; - const GUEST_AGENT_RPC_TIMEOUT: Duration = Duration::from_secs(30); impl App { @@ -367,7 +348,6 @@ impl App { config: Arc::new(config), pull_status: Arc::new(Mutex::new(std::collections::HashMap::new())), launch_locks: Arc::new(Mutex::new(HashMap::new())), - netd_probe: Arc::new(Mutex::new(None)), } } @@ -579,14 +559,12 @@ impl App { ) { Ok(processes) => processes, Err(error) => { - self.release_vm_interfaces(&vm_config.manifest.id, &runtime_networks) - .await; + self.release_vm_interfaces(&vm_config.manifest.id).await; return Err(error); } }; if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) { - self.release_vm_interfaces(&vm_config.manifest.id, &runtime_networks) - .await; + self.release_vm_interfaces(&vm_config.manifest.id).await; return Err(error); } { @@ -596,8 +574,7 @@ impl App { } for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { - self.release_vm_interfaces(&vm_config.manifest.id, &runtime_networks) - .await; + self.release_vm_interfaces(&vm_config.manifest.id).await; if let Err(clear_err) = work_dir.clear_runtime_networks() { warn!( id, @@ -639,14 +616,13 @@ impl App { let _launch = self.launch_lock(id).await; self.set_started(id, false)?; self.stop_vm_process(id).await?; - let networks = self.work_dir(id)?.runtime_networks(); // Not fallible: a VM that has been asked to stop is stopped whether or // not netd could be reached. What is left behind is reclaimed by this // VM's next launch, which releases before it prepares, or by its // removal. Not by reconciliation -- a stopped VM is still one this // instance claims, so a VM that is never started or removed again keeps // its interfaces. See [`App::release_vm_interfaces`]. - self.release_vm_interfaces(id, &networks).await; + self.release_vm_interfaces(id).await; Ok(()) } @@ -681,7 +657,7 @@ impl App { // it no longer wants an interface, and gating it on wanting one is how // the interfaces it left behind would become unreachable to every // later launch. - self.release_vm_interfaces(&vm.manifest.id, networks).await; + self.release_vm_interfaces(&vm.manifest.id).await; if !networks.iter().any(needs_netd_interface) { return Ok(()); } @@ -827,51 +803,6 @@ impl App { } Ok(()) })(); - // Ports asked for and not answered for used to vanish in silence: - // no warning, and `GetInfo` still listing them as though they - // worked. - let asked = &ingress[nic_index]; - match &response.ingress { - // A netd that forwards says what it built, so saying nothing is - // how one that does not reports it. A warning rather than a - // refusal: a VM deployed before this has been running with its - // ports dropped, and failing its launch now would turn a silent - // misconfiguration into an outage on upgrade. - None if !asked.is_empty() => warn!( - vm_id = %vm.manifest.id, - ports = asked.len(), - "netd on this node does not forward host ports, so this VM's port mappings \ - on interface {nic_index} are not published" - ), - // It answered, so hold it to the answer. A netd may refuse one - // port out of a set -- a node policy over which ports may be - // handed out is its own to state -- and the mapping that lost - // is the one worth naming. - Some(bound) => { - // What was answered, not what was asked. `GetInfo` reports - // the difference rather than the request. - network.ingress = bound.clone(); - for request in asked { - if !bound.iter().any(|binding| { - binding.answers( - &request.protocol, - &request.host_address, - request.host_port, - request.guest_port, - ) - }) { - warn!( - vm_id = %vm.manifest.id, - "netd did not publish {} {}:{} on interface {nic_index}", - request.protocol, - request.host_address, - request.host_port, - ); - } - } - } - None => {} - } if let Err(error) = accepted { self.roll_back_prepared_networks(prepared).await; return Err(error); @@ -952,38 +883,6 @@ impl App { } } - /// What this node's netd can do, cached for [`NETD_PROBE_TTL`]. - /// - /// Asked rather than inferred, and asked once per window rather than per - /// VM: it is a round trip, and the paths that want it are the ones already - /// making one. - /// An unreachable answer is not cached -- a failed connect costs nothing, - /// and holding on to it would keep a VMM blind to the netd an operator - /// just started. - pub(crate) async fn netd_capabilities(&self) -> netd::Reachability { - if let Some((_, reachability)) = self - .netd_probe - .lock() - .or_panic("mutex poisoned") - .as_ref() - .filter(|(asked, _)| asked.elapsed() < NETD_PROBE_TTL) - { - return reachability.clone(); - } - let reachability = netd::probe(&self.config.netd.socket).await; - // Only a real answer is cached. "Unreachable" and "too old to answer" - // are both produced by transient failures too, and holding either for - // half a minute turns one blip into a deployment refused for a reason - // that is not true -- `refuse_unpublishable_ports` reads this. A netd - // that genuinely predates `hello` is re-asked on a path that was - // already making a round trip. - if matches!(reachability, netd::Reachability::Capable(_)) { - *self.netd_probe.lock().or_panic("mutex poisoned") = - Some((std::time::Instant::now(), reachability.clone())); - } - reachability - } - /// Every VM whose interfaces this instance may still be using. /// /// Wider than the VMs it managed to load. A VM whose manifest is corrupt @@ -1087,16 +986,10 @@ impl App { return; } Err(error) => { - // It answered, so ask whether the answer means it cannot do - // this at all -- the same reading a release gets. - if self.netd_capabilities().await.supports("list") { - warn!("failed to list netd-managed interfaces: {error:#}"); - } else { - warn!( - "netd on this node cannot say what it holds, so interfaces belonging to a \ - VM removed while this VMM was down stay until the node runs a newer netd" - ); - } + warn!( + "failed to list netd-managed interfaces, so nothing is collected this pass: \ + {error:#}" + ); return; } }; @@ -1120,13 +1013,8 @@ impl App { } } if unattributed > 0 { - // Why there are any is the difference between "these are from - // before the upgrade and will sort themselves out" and "this netd - // never records ownership, so they never will". - let records_ownership = self.netd_capabilities().await.records_ownership(); info!( unattributed, - records_ownership, "host interfaces carry no ownership record, so no VMM can tell whose they are \ and none will collect them; `dstack-vmm netd list` shows them and \ `netd remove-interface` removes one" @@ -1150,7 +1038,7 @@ impl App { continue; } info!(vm_id = %vm_id, "collecting host interfaces no VM of this instance claims"); - self.release_vm_interfaces(&vm_id, &[]).await; + self.release_vm_interfaces(&vm_id).await; } } @@ -1180,7 +1068,7 @@ impl App { /// prepares, and startup reconciliation collects what no launch will ever /// reach. `recorded` is not the source of truth -- it is what a netd too /// old to sweep by identity has to be told instead. - pub(crate) async fn release_vm_interfaces(&self, vm_id: &str, recorded: &[Networking]) { + pub(crate) async fn release_vm_interfaces(&self, vm_id: &str) { // Ask for the release, rather than asking whether it can be asked for. // A probe first would put a second round trip in front of every stop // and -- worse -- would make a *busy* netd look like an absent one and @@ -1219,57 +1107,12 @@ impl App { debug!(vm_id, %error, "no netd to release interfaces from") } Err(error) => { - // It answered, so it is up. Now the question is worth a round - // trip: an operation a netd does not have looks exactly like - // one that failed, and only one of those has a fallback. A netd - // that has this operation and failed at it is a failure to - // report, not a reason to send it eight more requests. - if self.netd_capabilities().await.supports("remove_all") { - warn!( - vm_id, - "failed to release netd-managed interfaces: {error:#}" - ); - } else { - self.release_recorded_interfaces(vm_id, recorded).await; - } - } - } - } - - /// The teardown a netd that cannot sweep by identity gets. - /// - /// One `Remove` per index, over the record plus a margin for what the - /// record has lost. See [`LEGACY_TEARDOWN_SPAN`]. - async fn release_recorded_interfaces(&self, vm_id: &str, recorded: &[Networking]) { - warn!( - vm_id, - "netd on this node cannot release a VM's interfaces by identity; falling back to \ - the recorded ones. Interfaces it has no record of are left behind until this node \ - runs a netd that can sweep" - ); - for nic_index in (0..recorded.len().max(LEGACY_TEARDOWN_SPAN)).rev() { - // Beyond the record there is nothing to say whether the interface - // carried a binding, and an unfiltered removal still clears one it - // finds. Inside it, say what was built. - let filtered = recorded - .get(nic_index) - .and_then(|network| netd_teardown(network, &self.config.cvm)) - .unwrap_or(false); - let identity = InterfaceIdentity { - instance_id: self.config.cvm.instance_id.clone(), - vm_id: vm_id.to_string(), - nic_index, - }; - if let Err(error) = netd::request( - &self.config.netd.socket, - &NetdRequest::Remove { identity, filtered }, - ) - .await - { - if netd::is_unreachable(&error) { - return; - } - warn!(vm_id, nic_index, "failed to release interface: {error:#}"); + // Whatever this VM holds stays until it launches again, or + // until reconciliation collects it once nothing claims it. + warn!( + vm_id, + "failed to release netd-managed interfaces: {error:#}" + ); } } } @@ -1397,8 +1240,7 @@ impl App { } } - let runtime_networks = self.work_dir(id)?.runtime_networks(); - self.release_vm_interfaces(id, &runtime_networks).await; + self.release_vm_interfaces(id).await; // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. @@ -2584,49 +2426,35 @@ mod tests { let app = app_talking_to(Path::new("/nonexistent/dstack-netd.sock")); // Returns rather than propagating: there is no error type here on // purpose, because there is no caller that should act on one. - app.release_vm_interfaces("vm-1", &[]).await; + app.release_vm_interfaces("vm-1").await; } - /// The regression this pair exists to prevent: `remove_all` is an - /// operation, and an operation a netd does not have answers with an error - /// that looks exactly like the sweep having failed. Reading that as failure - /// used to fail the stop *and* leave every interface behind -- strictly - /// worse than the per-NIC removal it replaced. + /// A netd too old for the sweep refuses it, and a refusal is not a reason + /// to fail the stop. What it holds stays until this VM launches again or + /// until reconciliation collects it, which is what an operator upgrading + /// netd gets for free. #[tokio::test] - async fn a_netd_too_old_to_sweep_gets_the_teardown_it_understands() { + async fn a_netd_that_refuses_the_sweep_does_not_fail_the_stop() { let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); let app = app_talking_to(netd.socket()); - app.release_vm_interfaces("vm-1", &[]).await; - - let operations = netd.operations(); - assert_eq!( - operations[0], "remove_all", - "the release is asked for, not asked about" - ); - assert_eq!( - operations[1], "hello", - "only a refusal is worth a question, and then it is asked rather than inferred" - ); + app.release_vm_interfaces("vm-1").await; assert_eq!( - operations.iter().filter(|op| *op == "remove").count(), - LEGACY_TEARDOWN_SPAN, - "the record is not the only thing reached, even on a netd that cannot sweep" + netd.operations(), + vec!["remove_all"], + "asked once, and not asked about afterwards" ); } - /// A probe in front of every stop is a second round trip on the hot path, - /// and one whose failure mode is silence: a netd too slow to answer it - /// reads as an absent one, and an absent netd's release is skipped. The - /// operation itself cannot be misread that way. + /// One request, no question in front of it. A probe on the hot path is a + /// second round trip whose failure mode is silence: a netd too slow to + /// answer it reads as an absent one, and an absent netd's release is + /// skipped. #[tokio::test] - async fn a_netd_that_sweeps_is_asked_to_without_a_question_first() { - let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ - "hello", - "remove", - "remove_all", - ])); + async fn a_stop_asks_netd_to_sweep_without_a_question_first() { + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); let app = app_talking_to(netd.socket()); - app.release_vm_interfaces("vm-1", &[]).await; + app.release_vm_interfaces("vm-1").await; assert_eq!(netd.operations(), vec!["remove_all"]); let sweep = &netd.seen()[0]; @@ -2637,30 +2465,6 @@ mod tests { assert!(sweep.get("nic_index").is_none()); } - /// A real answer is reused; anything else is asked again. - /// - /// "Unreachable" and "too old to answer" are both produced by transient - /// failures too, and holding either for half a minute turns one blip into - /// a deployment refused for a reason that is not true. - #[tokio::test] - async fn only_a_real_capability_answer_is_reused() { - let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&["hello"])); - let app = app_talking_to(netd.socket()); - app.netd_capabilities().await; - app.netd_capabilities().await; - assert_eq!(netd.operations(), vec!["hello"], "asked once, read twice"); - - let legacy = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); - let app = app_talking_to(legacy.socket()); - app.netd_capabilities().await; - app.netd_capabilities().await; - assert_eq!( - legacy.operations().len(), - 2, - "an answer that may have been a blip is not held onto" - ); - } - fn held(tap: &str, instance: Option<&str>, vm: Option<&str>) -> serde_json::Value { serde_json::json!({ "tap": tap, @@ -2680,7 +2484,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); std::fs::create_dir(dir.path().join("live-vm")).unwrap(); let netd = netd::testing::FakeNetd::spawn_holding( - netd::testing::Behavior::capable(&["hello", "list", "remove_all"]), + netd::testing::Behavior::handling(&["list", "remove_all"]), vec![ held("dt000000000001", Some("test-instance"), Some("live-vm")), held("dt000000000002", Some("test-instance"), Some("dead-vm")), @@ -2715,10 +2519,8 @@ mod tests { #[tokio::test] async fn a_netd_that_cannot_enumerate_collects_nothing() { let dir = tempfile::tempdir().unwrap(); - let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::capable(&[ - "hello", - "remove_all", - ])); + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); let app = App::new( test_config(netd.socket(), dir.path()), SupervisorClient::new("http://127.0.0.1:0"), @@ -2738,7 +2540,7 @@ mod tests { #[tokio::test] async fn an_unreadable_vm_directory_collects_nothing() { let netd = netd::testing::FakeNetd::spawn_holding( - netd::testing::Behavior::capable(&["hello", "list", "remove_all"]), + netd::testing::Behavior::handling(&["list", "remove_all"]), vec![held( "dt000000000001", Some("test-instance"), diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index db5d00792..05fb6e42e 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -41,7 +41,6 @@ pub(crate) fn resolve_networking( // node configuration that names either is rejected at startup. resolved.netd_interface = crate::config::NetdInterface::None; resolved.device.clear(); - resolved.ingress.clear(); if !networking.bridge.is_empty() { resolved.nic.bridge = networking.bridge.clone(); } diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index f41e4ce8c..48af6f864 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -126,35 +126,6 @@ fn sanitize_optional>(value: Option) -> Option { value.filter(|value| !value.as_ref().trim().is_empty()) } -/// Whether one mapping's host port is actually reachable at the guest. -/// -/// A mapping is a request, and which NIC carries it decides who answers it: -/// QEMU's `hostfwd` on a user-mode NIC, always; the node's netd on a bridge -/// NIC, which may not forward host ports at all. A mapping naming a NIC the VM -/// no longer has is answered by nobody. Reporting the request as the answer is -/// how a VM could list published ports that nothing on the host forwarded. -fn published_at(mapping: &crate::app::PortMapping, networks: &[Networking]) -> bool { - let Some(nic_index) = crate::app::network::ingress_nic(mapping, networks) else { - return false; - }; - let Some(network) = networks.get(nic_index) else { - return false; - }; - if network.nic.mode == crate::config::NetworkingMode::User { - // QEMU carries these itself, for as long as it is up. - return true; - } - let host_address = mapping.address.to_string(); - network.ingress.iter().any(|binding| { - binding.answers( - mapping.protocol.as_str(), - &host_address, - mapping.from, - mapping.to, - ) - }) -} - impl VmInfo { /// Takes no `CvmConfig` on purpose. Everything it reports about a VM's /// data plane was decided when that VM launched and written into @@ -227,9 +198,6 @@ impl VmInfo { .port_map .iter() .map(|mapping| pb::PortMapping { - published: self - .running - .then(|| published_at(mapping, effective_networks)), nic_index: mapping.nic_index.map(|index| index as u32), protocol: mapping.protocol.as_str().into(), host_address: mapping.address.to_string(), @@ -360,86 +328,8 @@ impl VmState { #[cfg(test)] mod tests { - use super::{interfaces_to_proto, networking_to_proto, published_at, sanitize_optional}; - use crate::app::PortMapping; - use crate::config::{Networking, NetworkingMode, NicNetworking, Protocol}; - use crate::netd::IngressBinding; - - fn nic(mode: NetworkingMode) -> Networking { - Networking { - nic: NicNetworking { - mode, - ..NicNetworking::default() - }, - ..Networking::default() - } - } - - fn mapping(host_port: u16, nic_index: Option) -> PortMapping { - PortMapping { - address: "0.0.0.0".parse().unwrap(), - protocol: Protocol::Tcp, - from: host_port, - to: 8080, - nic_index, - } - } - - /// A mapping is a request. Which NIC carries it decides who answers it, - /// and on a bridge the answer comes from a netd that may not forward host - /// ports at all -- so reporting the request as the answer is how a VM - /// could list published ports nothing on the host forwarded. - #[test] - fn a_port_is_reported_published_only_where_something_publishes_it() { - // QEMU carries a user-mode NIC's mappings itself. - let networks = [nic(NetworkingMode::User)]; - assert!(published_at(&mapping(443, None), &networks)); - - // A bridge NIC's are netd's to publish, and this node's netd said - // nothing about them. - let networks = [nic(NetworkingMode::Bridge)]; - assert!(!published_at(&mapping(443, None), &networks)); - - // Until it does. - let mut published = nic(NetworkingMode::Bridge); - published.ingress = vec![IngressBinding { - protocol: "tcp".into(), - host_address: "0.0.0.0".into(), - host_port: 443, - guest_port: 8080, - }]; - let networks = [published]; - assert!(published_at(&mapping(443, None), &networks)); - // Answered for one port is not answered for another. - assert!(!published_at(&mapping(444, None), &networks)); - - // A mapping naming a NIC the VM no longer has is answered by nobody. - assert!(!published_at(&mapping(443, Some(7)), &networks)); - - // An admin port on loopback and a published one differ only in the - // address, so a netd that narrowed 0.0.0.0 to 127.0.0.1 has not met - // the request -- and reporting it as met would report a port as - // reachable from the network when it is not. - let mut narrowed = nic(NetworkingMode::Bridge); - narrowed.ingress = vec![IngressBinding { - protocol: "tcp".into(), - host_address: "127.0.0.1".into(), - host_port: 443, - guest_port: 8080, - }]; - assert!(!published_at(&mapping(443, None), &[narrowed])); - - // An address netd did not state at all is a netd that echoes less than - // it was told, not one that narrowed anything. - let mut silent = nic(NetworkingMode::Bridge); - silent.ingress = vec![IngressBinding { - protocol: "tcp".into(), - host_address: String::new(), - host_port: 443, - guest_port: 8080, - }]; - assert!(published_at(&mapping(443, None), &[silent])); - } + use super::{interfaces_to_proto, networking_to_proto, sanitize_optional}; + use crate::config::{NetworkingMode, NicNetworking}; /// Custom mode hands the operator the whole netdev string and the VMM never /// parses it, so it has no data-plane state to report. Reporting the diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 194ae1a57..e6b55ae6f 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -1085,15 +1085,6 @@ pub struct Networking { /// the VM runs, and what has to be removed is what was created. #[serde(default, skip_serializing_if = "NetdInterface::is_none")] pub netd_interface: NetdInterface, - /// The host ports netd actually published for this NIC. - /// - /// Runtime state, like `device`: resolution always clears it. What was - /// asked for lives in the manifest; this is what was answered, and the - /// difference between the two is the whole reason to keep it. Reporting - /// the request as though it were the answer is how a VM's ports could be - /// listed as published while nothing on the host forwarded them. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub ingress: Vec, } /// The host interface netd created for a NIC, if any. diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 98295ca9b..f1a72c59f 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -24,11 +24,10 @@ use path_absolutize::Absolutize; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; -use crate::app::network::ingress_nic; use crate::app::{ - mode_carries_ingress, needs_swtpm, resolve_networking, resolved_networks, - validate_resolved_network, validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, - Manifest, PortMapping, VmWorkDir, + mode_carries_ingress, needs_swtpm, resolve_networking, validate_resolved_network, + validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, + VmWorkDir, }; use crate::config::{CvmConfig, Networking, NetworkingMode, NicNetworking}; @@ -200,8 +199,22 @@ fn resolved_nic_modes( /// warning either. fn validate_port_mapping_nics(mappings: &[PortMapping], modes: &[NetworkingMode]) -> Result<()> { let nic_count = modes.len(); + let has_ingress = modes.iter().any(|mode| mode_carries_ingress(*mode)); for mapping in mappings { let Some(index) = mapping.nic_index else { + // Unpinned, so it goes wherever the default resolves -- unless + // there is nowhere for it to resolve to. That is a property of the + // VM's own topology, which is why it is decided here and not by + // asking the host anything. + if !has_ingress { + bail!( + "port mapping {} {}:{} has no NIC to enter through: this VM has no user-mode \ + or bridge interface", + mapping.protocol.as_str(), + mapping.address, + mapping.from + ); + } continue; }; let Some(mode) = modes.get(index) else { @@ -832,71 +845,16 @@ impl RpcHandler { Ok(true) } - - /// Refuses a port mapping this node has no way to publish. - /// - /// At deployment, where refusing costs nothing: nothing is running on the - /// answer yet, and the alternative is a VM that reports published ports - /// nothing on the host forwards. A launch only warns about the same thing, - /// because a VM deployed before this has been running with its ports - /// dropped, and failing it on upgrade would turn a silent misconfiguration - /// into an outage. - /// - /// Two ways to have nowhere to go. A mapping can resolve to no NIC at all - /// -- a VM whose every NIC is macvtap or custom -- or to a bridge NIC on a - /// node whose netd builds interfaces and does not forward host ports, which - /// is what the netd in this repository does. - async fn refuse_unpublishable_ports(&self, manifest: &Manifest) -> Result<()> { - let networks = resolved_networks(manifest, &self.app.config.cvm); - let mut needs_netd = Vec::new(); - for mapping in &manifest.port_map { - let backend = ingress_nic(mapping, &networks).and_then(|index| networks.get(index)); - let named = format!( - "{} {}:{}", - mapping.protocol.as_str(), - mapping.address, - mapping.from - ); - match backend { - None => bail!( - "port mapping {named} has no NIC to enter through: this VM has no user-mode \ - or bridge interface" - ), - // QEMU carries these itself. - Some(network) if network.nic.mode == NetworkingMode::User => {} - Some(_) => needs_netd.push(named), - } - } - if needs_netd.is_empty() { - return Ok(()); - } - let reachability = self.app.netd_capabilities().await; - if reachability.forwards_ingress() { - return Ok(()); - } - let named = needs_netd.join(", "); - // "It does not forward" and "it could not be asked" call for different - // fixes, and only one of them is about the deployment. - if !reachability.is_reachable() { - bail!( - "port mapping {named} enters through a bridge NIC, which only netd can publish, \ - and netd could not be reached to ask whether it does" - ); - } - bail!( - "port mapping {named} enters through a bridge NIC, which only netd can publish, and \ - the netd on this node does not forward host ports ({}). Put the mapping on a \ - user-mode NIC with @, or run a netd that forwards", - reachability.describe() - ) - } } impl VmmRpc for RpcHandler { async fn create_vm(self, request: VmConfiguration) -> Result { let manifest = create_manifest_from_vm_config(request.clone(), &self.app.config.cvm)?; self.validate_port_mapping_conflicts(None, &manifest.port_map)?; - self.refuse_unpublishable_ports(&manifest).await?; + validate_port_mapping_nics( + &manifest.port_map, + &resolved_nic_modes(&manifest.networks, &self.app.config.cvm, manifest.vcpu), + )?; let id = manifest.id.clone(); info!(vm_id = %id, "create_vm RPC called"); let app_id = manifest.app_id.clone(); @@ -1096,10 +1054,7 @@ impl VmmRpc for RpcHandler { .await? .is_some_and(|info| info.state.status.is_running()); if !is_running { - let runtime_networks = vm_work_dir.runtime_networks(); - self.app - .release_vm_interfaces(&request.id, &runtime_networks) - .await; + self.app.release_vm_interfaces(&request.id).await; vm_work_dir.clear_runtime_networks()?; } manifest.networks = networks; @@ -1116,7 +1071,6 @@ impl VmmRpc for RpcHandler { &manifest.port_map, &resolved_nic_modes(&manifest.networks, &self.app.config.cvm, manifest.vcpu), )?; - self.refuse_unpublishable_ports(&manifest).await?; } let compose_file = fs::read_to_string(vm_work_dir.app_compose_path()) .context("failed to read app compose for swtpm decision")?; diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 5044696b1..6422aec25 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -37,8 +37,6 @@ use crate::config::{NetdConfig, NetworkFilterConfig}; const MAX_MESSAGE_SIZE: u64 = 64 * 1024; const CONNECTION_TIMEOUT: Duration = Duration::from_secs(35); -/// How long a capability probe waits. See [`probe`]. -const PROBE_TIMEOUT: Duration = Duration::from_secs(5); const COMMAND_TIMEOUT: Duration = Duration::from_secs(30); const IP_PATH: &str = "/usr/sbin/ip"; const VIRSH_PATH: &str = "/usr/bin/virsh"; @@ -50,17 +48,6 @@ const MAX_QUEUES: u32 = 64; /// whole-VM sweep has to enumerate, since it derives names instead of reading a /// record. const MAX_NIC_INDEX: usize = 255; -/// Every operation this netd accepts, answered to `hello`. -const OPERATIONS: &[&str] = &[ - "hello", - "prepare_bridge", - "prepare_macvtap", - "remove", - "remove_all", - "list", - "remove_interface", - "check", -]; /// The interface names netd may create. Reserved: anything matching it is /// netd's to delete, and nothing else on the host may take one. const TAP_PREFIX: &str = "dt"; @@ -164,16 +151,6 @@ pub struct IngressRequest { pub guest_port: u16, } -/// One forwarding rule a netd established, echoed so the caller can report what -/// the VM actually got rather than what it asked for. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct IngressBinding { - pub protocol: String, - pub host_address: String, - pub host_port: u16, - pub guest_port: u16, -} - /// One host resource netd holds. /// /// `instance_id` and `vm_id` are absent when the interface carries no record @@ -200,69 +177,6 @@ pub struct InterfaceRecord { pub bound: bool, } -/// What a netd can do, asked rather than inferred from a failure. -/// -/// `queues` and `ingress` report a *missing feature* by leaving a response -/// field out, which works because both ride on an operation every netd has. An -/// operation a netd does not have cannot answer that way: it fails, and a -/// failure is indistinguishable from the operation failing for a real reason. -/// A caller left to guess from the message either turns a missing feature into -/// an outage or turns a real failure into silence -- and teardown, where the -/// consequence of guessing wrong is a leaked host interface, is exactly where -/// neither is acceptable. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Capabilities { - /// Implementation and version, for the operator's log. Never parsed: - /// `operations` is what decisions are made from. - #[serde(default)] - pub version: String, - /// Every operation this netd accepts. - #[serde(default)] - pub operations: Vec, - /// Whether it forwards host ports at all. A prepare's `ingress` field still - /// says what one interface actually got; this says whether asking is - /// meaningful, which is what deployment has to know before any prepare - /// exists to read. - #[serde(default)] - pub ingress: bool, - /// Whether it records ownership on the interface itself. A whole-host - /// collection cannot tell one VMM instance's interfaces from another's - /// without it, so a netd that says no is never asked to act on the - /// *absence* of a record: "no record" would describe every interface on - /// the host, including another instance's running VMs. - #[serde(default)] - pub attribution: bool, -} - -impl Capabilities { - pub fn supports(&self, operation: &str) -> bool { - self.operations.iter().any(|name| name == operation) - } -} - -impl IngressBinding { - /// Whether this binding answers a request for one host port. - /// - /// The address is part of the answer, not decoration: an admin port bound - /// to loopback and a published one differ only there, so a netd that - /// narrowed `0.0.0.0` to `127.0.0.1` has not met the request, and - /// reporting it as met would report a port as reachable from the network - /// when it is not. An address netd did not state at all is not a narrowing; - /// it is a netd that echoes less than it was told. - pub fn answers( - &self, - protocol: &str, - host_address: &str, - host_port: u16, - guest_port: u16, - ) -> bool { - self.protocol == protocol - && self.host_port == host_port - && self.guest_port == guest_port - && (self.host_address.is_empty() || self.host_address == host_address) - } -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrepareMacvtapRequest { #[serde(flatten)] @@ -285,18 +199,8 @@ pub struct PrepareMacvtapRequest { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "operation", rename_all = "snake_case")] pub enum Request { - /// Ask what this netd can do, before asking it to do anything. - /// - /// Cheap by construction: answered without taking the operation lock, so a - /// caller learns what netd can do without first waiting for what it is - /// doing. It doubles as the liveness probe -- a netd that answers is up, - /// and one that predates this answers an error, which is still an answer. - Hello, PrepareBridge(PrepareBridgeRequest), PrepareMacvtap(PrepareMacvtapRequest), - /// - /// Releases everything the interface owns, including any host ports a - /// forwarding netd published for it. See [`PrepareBridgeRequest::ingress`]. Remove { #[serde(flatten)] identity: InterfaceIdentity, @@ -375,15 +279,9 @@ struct Response { /// forward host ports, which is how the caller tells "nothing was asked /// for" apart from "this request was ignored" -- the same reading `queues` /// gets above. - #[serde(default, skip_serializing_if = "Option::is_none")] - ingress: Option>, /// How many interfaces a whole-VM sweep deleted. #[serde(default, skip_serializing_if = "Option::is_none")] removed: Option, - /// What this netd can do. Absent from one that predates `hello`, which is - /// the same reading `queues` and `ingress` get. - #[serde(default, skip_serializing_if = "Option::is_none")] - capabilities: Option, /// Whether the pass stopped on its deadline with work left. See /// [`COLLECTION_DEADLINE`]. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -410,14 +308,12 @@ enum Outcome { tap: String, device: Option, queues: Option, - ingress: Option>, }, /// A sweep names no single interface, so it reports how many it deleted. Swept { removed: usize, incomplete: bool, }, - Hello(Capabilities), Listed(Vec), } @@ -427,7 +323,6 @@ impl Outcome { tap, device: None, queues: None, - ingress: None, } } @@ -437,9 +332,7 @@ impl Outcome { tap: None, device: None, queues: None, - ingress: None, removed: None, - capabilities: None, incomplete: None, interfaces: None, error: None, @@ -449,12 +342,10 @@ impl Outcome { tap, device, queues, - ingress, } => { response.tap = Some(tap); response.device = device; response.queues = queues; - response.ingress = ingress; } Self::Swept { removed, @@ -463,7 +354,6 @@ impl Outcome { response.removed = Some(removed); response.incomplete = Some(incomplete); } - Self::Hello(capabilities) => response.capabilities = Some(capabilities), Self::Listed(interfaces) => response.interfaces = Some(interfaces), } response @@ -565,8 +455,6 @@ pub fn instance_id(configured: &str, run_path: &Path) -> String { pub struct PreparedInterface { pub device: Option, pub queues: Option, - /// The forwarding rules netd established, if it forwards host ports at all. - pub ingress: Option>, } /// Marker carried in the error chain when the VMM could not reach netd at all. @@ -604,7 +492,6 @@ pub async fn request(socket: &Path, request: &Request) -> Result Result Result { let request = Request::RemoveAll { instance_id: instance_id.to_string(), @@ -662,97 +549,8 @@ pub async fn list(socket: &Path, instance_id: &str) -> Result bool { - !matches!(self, Self::Unreachable) - } - - pub fn supports(&self, operation: &str) -> bool { - match self { - Self::Unreachable | Self::Legacy => false, - Self::Capable(capabilities) => capabilities.supports(operation), - } - } - - /// Whether it records which VM an interface belongs to. Unknown counts as - /// no, for the same reason `forwards_ingress` does. - pub fn records_ownership(&self) -> bool { - matches!(self, Self::Capable(capabilities) if capabilities.attribution) - } - - /// Whether asking this netd to forward host ports is meaningful. Unknown - /// counts as no: a caller that assumed yes would report ports as published - /// on the strength of never having asked. - pub fn forwards_ingress(&self) -> bool { - matches!(self, Self::Capable(capabilities) if capabilities.ingress) - } - - pub fn describe(&self) -> String { - match self { - Self::Unreachable => "unreachable".to_string(), - Self::Legacy => "reachable, predates capability reporting".to_string(), - Self::Capable(capabilities) => { - format!( - "{} [{}]", - capabilities.version, - capabilities.operations.join(" ") - ) - } - } - } -} - -/// Asks what this node's netd can do. -/// -/// Never fails: not being there, and being there but too old to say, are both -/// answers a caller has to act on rather than propagate. Both are also -/// distinguishable here and nowhere else -- an error from any other operation -/// cannot tell "netd refused this" from "netd does not have this". -pub async fn probe(socket: &Path) -> Reachability { - // Bounded well below the request timeout. Every teardown asks, and a netd - // that accepts a connection and then stops answering must not turn each of - // them into a thirty-second stall; not answering promptly is, for the - // caller's purposes, the same as not being there. - let answer = match timeout(PROBE_TIMEOUT, exchange(socket, &Request::Hello)).await { - Ok(answer) => answer, - Err(_) => { - warn!("netd accepted a connection but did not answer hello in time"); - return Reachability::Unreachable; - } - }; - match answer { - Ok(response) => match response.capabilities { - Some(capabilities) => Reachability::Capable(capabilities), - // It answered `hello` with nothing to say, which is not a shape - // this netd produces. Treat it as the older protocol rather than - // trusting an empty capability set. - None => Reachability::Legacy, - }, - // It answered, so it is up; it just does not know the question. - Err(error) if !is_unreachable(&error) => { - debug!("netd does not answer hello: {error:#}"); - Reachability::Legacy - } - Err(_) => Reachability::Unreachable, - } -} - async fn exchange(socket: &Path, request: &Request) -> Result { let operation = match request { - Request::Hello => "hello", Request::PrepareBridge(_) => "prepare_bridge", Request::PrepareMacvtap(_) => "prepare_macvtap", Request::Remove { .. } => "remove", @@ -907,12 +705,6 @@ async fn serve_connection( debug!("netd liveness probe"); return Ok(()); } - // Answered here rather than in `handle_request`, which reaches the - // blocking pool first. That pool is finite and its tasks cannot be - // cancelled, so a node whose `virsh` calls are all timing out fills it - // -- and `hello` queued behind them times out too, putting back the - // "a busy netd reads as an absent one" this daemon exists to avoid. - Ok(Some(Request::Hello)) => Ok(Outcome::Hello(capabilities())), Ok(Some(request)) => { // `handle_request` shells out to `ip` and `virsh` and waits on the // operation lock, so it can block for as long as those take. On a @@ -940,9 +732,7 @@ async fn serve_connection( tap: None, device: None, queues: None, - ingress: None, removed: None, - capabilities: None, incomplete: None, interfaces: None, error: Some(format!("{error:#}")), @@ -977,31 +767,10 @@ async fn read_request(stream: &mut UnixStream) -> Result> { .context("invalid netd request") } -/// What this build of netd can do. See [`Capabilities`]. -fn capabilities() -> Capabilities { - Capabilities { - version: format!("dstack-netd {}", env!("CARGO_PKG_VERSION")), - operations: OPERATIONS.iter().map(|name| name.to_string()).collect(), - // This netd builds interfaces; it is not the host's forwarder. The - // response field says so per prepare; this says so before one. - ingress: false, - attribution: true, - } -} - fn handle_request(config: &NetdConfig, request: Request) -> Result { - // Answered before the lock. A caller asks this to find out whether netd - // can do the thing it is about to ask for, and making it wait behind a - // running collection would put a whole-host sweep in front of every - // launch's first question. - if matches!(request, Request::Hello) { - return Ok(Outcome::Hello(capabilities())); - } let libvirt_uri = config.libvirt_uri.as_str(); let _lock = OperationLock::acquire()?; match request { - // Handled above, before the lock. - Request::Hello => Ok(Outcome::Hello(capabilities())), Request::PrepareBridge(request) => { prepare_bridge(libvirt_uri, &request, config.filter_policy()) } @@ -1136,7 +905,6 @@ fn prepare_macvtap( tap, device: Some(device), queues: Some(queues), - ingress: None, }) } Err(error) => { @@ -1225,7 +993,6 @@ fn prepare_bridge( // This netd builds interfaces; it is not the host's forwarder. Saying // nothing here is what tells the caller that, so ports it asked for are // reported as unmet rather than assumed done. - ingress: None, }) } @@ -1803,37 +1570,16 @@ pub(crate) mod testing { /// How the fake answers. #[derive(Debug, Clone)] pub(crate) enum Behavior { - /// Answers `hello` with the given operations, and every listed - /// operation with a plausible success. - Capable { - operations: Vec, - ingress: bool, - /// Whether it claims to record who an interface belongs to. - attribution: bool, - }, - /// Reached, but predates `hello`: every unknown operation is an error, - /// exactly as `serde` produces one. + /// Answers every listed operation with a plausible success, and + /// anything else with the error `serde` produces for an unknown one. + Handles(Vec), + /// An older netd: every operation this one added is an error. Legacy, } impl Behavior { - pub(crate) fn capable(operations: &[&str]) -> Self { - Self::Capable { - operations: operations.iter().map(|name| name.to_string()).collect(), - ingress: false, - attribution: true, - } - } - - pub(crate) fn forwarding(operations: &[&str]) -> Self { - match Self::capable(operations) { - Self::Capable { operations, .. } => Self::Capable { - operations, - ingress: true, - attribution: true, - }, - other => other, - } + pub(crate) fn handling(operations: &[&str]) -> Self { + Self::Handles(operations.iter().map(|name| name.to_string()).collect()) } } @@ -1911,7 +1657,7 @@ pub(crate) mod testing { fn answer(behavior: &Behavior, interfaces: &[Value], request: &Value) -> Value { let operation = request["operation"].as_str().unwrap_or_default(); - let (operations, ingress, attribution) = match behavior { + let operations = match behavior { Behavior::Legacy => { return match operation { // What the real thing answers for an operation it knows. @@ -1924,11 +1670,7 @@ pub(crate) mod testing { }), }; } - Behavior::Capable { - operations, - ingress, - attribution, - } => (operations, *ingress, *attribution), + Behavior::Handles(operations) => operations, }; if !operations.iter().any(|name| name == operation) { return json!({ @@ -1937,27 +1679,11 @@ pub(crate) mod testing { }); } match operation { - "hello" => json!({ + "prepare_bridge" | "prepare_macvtap" => json!({ "ok": true, - "capabilities": { - "version": "fake-netd", - "operations": operations, - "ingress": ingress, - "attribution": attribution, - }, + "tap": "dtdeadbeef00", + "queues": request["queues"].as_u64().unwrap_or(1).max(1), }), - "prepare_bridge" | "prepare_macvtap" => { - let mut response = json!({ - "ok": true, - "tap": "dtdeadbeef00", - "queues": request["queues"].as_u64().unwrap_or(1).max(1), - }); - if ingress { - let asked = request["ingress"].as_array().cloned().unwrap_or_default(); - response["ingress"] = Value::Array(asked); - } - response - } "remove" | "check" => json!({"ok": true, "tap": "dtdeadbeef00"}), "remove_all" => json!({"ok": true, "removed": 0, "incomplete": false}), "list" => { @@ -2416,32 +2142,6 @@ mod tests { assert!(decoded.ingress.is_empty()); } - #[test] - fn saying_nothing_about_ports_is_how_a_netd_reports_it_forwards_none() { - // The same reading `queues` gets: absent distinguishes "this netd does - // not do that" from "nothing was asked for", so ports are never assumed - // forwarded just because the TAP came back. - let response: Response = serde_json::from_value(serde_json::json!({ - "ok": true, - "tap": "dt000000000000", - })) - .unwrap(); - assert!(response.ingress.is_none()); - - let response: Response = serde_json::from_value(serde_json::json!({ - "ok": true, - "tap": "dt000000000000", - "ingress": [{ - "protocol": "udp", - "host_address": "0.0.0.0", - "host_port": 7483, - "guest_port": 51820, - }], - })) - .unwrap(); - assert_eq!(response.ingress.unwrap().len(), 1); - } - /// A prepare carrying only the fields that predate this change. fn decode_minimal_bridge() -> Request { serde_json::from_value(serde_json::json!({ @@ -2703,13 +2403,13 @@ mod tests { #[tokio::test] async fn a_sweep_without_a_count_is_not_read_as_an_empty_one() { // Claims the operation, answers without the field. - let netd = testing::FakeNetd::spawn(testing::Behavior::capable(&["hello"])); + let netd = testing::FakeNetd::spawn(testing::Behavior::handling(&[])); let error = remove_all(netd.socket(), "instance", "vm") .await .expect_err("an answer with no count is not a successful sweep"); assert!(!is_unreachable(&error)); - let netd = testing::FakeNetd::spawn(testing::Behavior::capable(&["hello", "remove_all"])); + let netd = testing::FakeNetd::spawn(testing::Behavior::handling(&["remove_all"])); let sweep = remove_all(netd.socket(), "instance", "vm").await.unwrap(); assert_eq!(sweep.removed, 0); assert!(!sweep.incomplete); @@ -2730,52 +2430,4 @@ mod tests { .context("failed to prepare netd-managed networking"); assert!(!is_unreachable(&other)); } - - #[tokio::test] - async fn a_probe_tells_absent_from_old_from_answering() { - assert!(matches!( - probe(Path::new("/nonexistent/netd.sock")).await, - Reachability::Unreachable - )); - - let legacy = testing::FakeNetd::spawn(testing::Behavior::Legacy); - let reachability = probe(legacy.socket()).await; - assert!(matches!(reachability, Reachability::Legacy)); - // Reached, so a caller must not treat it as absent -- but it can do - // nothing this netd was not already able to do. - assert!(reachability.is_reachable()); - assert!(!reachability.supports("remove_all")); - assert!(!reachability.forwards_ingress()); - - let netd = - testing::FakeNetd::spawn(testing::Behavior::forwarding(&["hello", "remove_all"])); - let reachability = probe(netd.socket()).await; - assert!(reachability.supports("remove_all")); - assert!(!reachability.supports("gc")); - assert!(reachability.forwards_ingress()); - } - - /// The whole point of asking: a caller must be able to tell an operation - /// this netd does not have from one that failed, and the two look the same - /// in an error message. - #[test] - fn a_probe_reports_what_this_netd_can_do() { - let value = serde_json::to_value(&Request::Hello).unwrap(); - assert_eq!(value["operation"], "hello"); - - let response = Outcome::Hello(capabilities()).into_response(); - let capabilities = response.capabilities.expect("hello answers capabilities"); - assert!(capabilities.supports("remove_all")); - assert!(capabilities.supports("hello")); - assert!(!capabilities.supports("forward_the_whole_internet")); - // This netd builds interfaces and does not forward host ports. Saying - // so before a prepare is what lets deployment refuse a mapping it - // cannot honour, instead of a launch warning about it afterwards. - assert!(!capabilities.ingress); - assert!(capabilities.attribution); - - // Absent capabilities is an older netd, not one that can do nothing. - let legacy: Response = serde_json::from_str(r#"{"ok":true}"#).unwrap(); - assert!(legacy.capabilities.is_none()); - } } From 68e588a785b9878b3c5e6c91549dac491f97181a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 08:17:47 -0700 Subject: [PATCH 29/34] refactor(vmm): say that only QEMU publishes a host port, and stop pretending Three structures modelled a host that forwards bridge-NIC ports. Nothing on this host does. `PrepareBridgeRequest.ingress` and `IngressRequest` carried a per-NIC port list to netd, which read it exactly zero times. It was a declaration of intent to a daemon with no mechanism to honour it -- the request half of the `published` chain the previous commit removed from the response. With it gone, `mode_carries_ingress` can say what is true: QEMU's `hostfwd=`, and nothing else. That makes `default_ingress_nic` one line rather than a fallback to a bridge NIC that would have accepted a mapping and dropped it -- and a bridge pin is now refused at deployment, where the caller is there to be told, rather than resolving to a NIC with no path into the guest. The refusal is narrower in the other direction: an *unpinned* mapping is never refused, because a bridge-only VM with a port map deploys today and must keep deploying. What it strands, the launch names. `NetdInterface` and `netd_teardown` were a closed loop. At the base of this branch the record decided which per-NIC `Remove` teardown sent; this PR replaced that with `remove_all`, which derives names and needs no record. What was left wrote the field, persisted it, and read it only through `netd_teardown`, whose one production caller assigned the result straight back into the field it had just read. `is_filtered` had no callers at all. The 6-line comment justifying it still described the behaviour this PR deleted. `Request::Remove.filtered` was mandatory on decode and explicitly ignored by the handler, kept for "a netd that predates that reasoning". `origin/next` has no such field on `Remove`, so the compatibility was with a stacked branch rather than with anything released -- and serde ignores an unknown flattened key, so an in-flight old netd decodes a request without it either way. Also repairs two comments a `published: None` cleanup truncated mid-sentence, and three places -- a doc, a vmm.toml comment and a qemu.rs comment -- claiming netd "arbitrates host ports between VMM instances". --- docs/bridge-networking.md | 24 ++- dstack/crates/dstack-cli-core/src/ports.rs | 1 - dstack/crates/dstackup/src/install.rs | 1 - dstack/vmm/src/app.rs | 51 +----- dstack/vmm/src/app/network.rs | 187 +++------------------ dstack/vmm/src/app/qemu.rs | 8 +- dstack/vmm/src/config.rs | 49 +----- dstack/vmm/src/main_service.rs | 19 +-- dstack/vmm/src/netd.rs | 134 +-------------- dstack/vmm/vmm.toml | 6 +- 10 files changed, 58 insertions(+), 422 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 405b713a7..62abda89d 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -159,10 +159,9 @@ filesystem permissions authorize callers. This used to be conditional — `netd` built the TAP when libvirt filtering was on or when the NIC wanted more than one queue pair, and otherwise QEMU's setuid `qemu-bridge-helper` did. Two owners meant two answers to the same questions: -which netdev QEMU gets, whether vhost is really on, and where a bridge NIC's -published ports go. Only `netd` can answer the last one, because only `netd` -sees every VMM instance on the host and can arbitrate a port between them. So a -bridge NIC's host interface has one owner now, on every node. +which netdev QEMU gets, whether vhost is really on, and what a bridge NIC's TAP +is built with. So a bridge NIC's host interface has one owner now, on every +node. `qemu-bridge-helper` is no longer used, and `/etc/qemu/bridge.conf` no longer needs an `allow` line for the bridge. @@ -228,16 +227,15 @@ by both. ### Which ports a bridge NIC can publish -QEMU publishes a user-mode NIC's mappings itself, with `hostfwd=`. A bridge -NIC's would have to be published by `netd`, and **the `netd` in this repository -builds interfaces; it does not forward host ports.** A mapping that resolves to -a bridge NIC is carried to `netd` in the prepare and goes no further. +QEMU publishes a port with `hostfwd=` on a user-mode NIC, and that is the only +mechanism this host has. **The `netd` in this repository builds interfaces; it +does not forward host ports**, so a bridge NIC cannot carry a port mapping. -The VMM does not track whether the host is forwarding. It reports what a VM -asked for, and a mapping is refused only when the VM's own topology gives it -nowhere to go — a pinned NIC that is macvtap or does not exist, or a VM with no -user-mode and no bridge NIC at all. Those are facts about the VM, decided -without asking the host anything. +`--port …@` therefore only ever names a user-mode NIC. Pinning to a bridge, +macvtap or custom NIC is refused at deployment, where the caller is there to be +told. An unpinned mapping goes to the first user-mode NIC; a VM that has none is +not refused — it may have been deployed before this — but every mapping it +strands is named in the launch log. ## Who owns an interface diff --git a/dstack/crates/dstack-cli-core/src/ports.rs b/dstack/crates/dstack-cli-core/src/ports.rs index 86bca1c9c..142be1b16 100644 --- a/dstack/crates/dstack-cli-core/src/ports.rs +++ b/dstack/crates/dstack-cli-core/src/ports.rs @@ -77,7 +77,6 @@ pub fn parse_port(spec: &str) -> Result { host_port, vm_port, nic_index, - // A request, not an answer: the server reports what it actually }) } diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index 20852f749..b32f9fea0 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -297,7 +297,6 @@ pub(crate) async fn cmd_install(mut o: InstallOpts, release_api_base_url: &str) // Unpinned: this deploys the node default topology, which // is one NIC, and the VMM resolves that itself. nic_index: None, - // A request, not an answer: the VMM reports what it }], ..Default::default() }; diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 28715f3bb..2f629b469 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -3,10 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - config::{ - Config, NetdInterface, Networking, NetworkingMode, NicNetworking, ProcessAnnotation, - Protocol, - }, + config::{Config, Networking, NetworkingMode, NicNetworking, ProcessAnnotation, Protocol}, logrotate, netd::{ self, InterfaceIdentity, PrepareBridgeRequest, PrepareMacvtapRequest, @@ -45,9 +42,9 @@ use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; pub(crate) use network::{ - filters_bridge_traffic, ingress_for, mode_carries_ingress, needs_netd_interface, netd_teardown, - resolve_networking, resolved_networks, settle_vhost, stranded_ingress, - validate_resolved_network, validate_resolved_networks, + filters_bridge_traffic, mode_carries_ingress, needs_netd_interface, resolve_networking, + resolved_networks, settle_vhost, stranded_ingress, validate_resolved_network, + validate_resolved_networks, }; pub use qemu::VmConfig; // Exported so the RPC layer can assert that everything it reports is @@ -661,12 +658,6 @@ impl App { if !networks.iter().any(needs_netd_interface) { return Ok(()); } - // Resolved before the loop borrows `networks` mutably, and once rather - // than per NIC, so both the request and the warning below read the same - // answer. - let ingress: Vec> = (0..networks.len()) - .map(|nic_index| ingress_for(&vm.manifest.port_map, networks, nic_index)) - .collect(); let qemu_uid = Uid::effective().as_raw(); // Only ever read back out of a log line: netd is told where the VM // lives so an operator holding an opaque TAP name can reach the VM @@ -709,11 +700,6 @@ impl App { filtered, queues, workdir: workdir.clone(), - // Only the mappings that resolve to this NIC. One mapping - // lands on exactly one, and a user-mode NIC's are emitted - // as QEMU `hostfwd=` instead, so no host port is claimed - // twice. - ingress: ingress[nic_index].clone(), }), NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap(PrepareMacvtapRequest { identity: identity.clone(), @@ -738,7 +724,6 @@ impl App { &self.config.netd.socket, &NetdRequest::Remove { identity: identity.clone(), - filtered, }, ) .await @@ -768,15 +753,7 @@ impl App { }; } }; - prepared.push((identity.clone(), filtered)); - // netd built this one. Record it now, before anything else can - // fail, so teardown never has to re-derive it from a node - // configuration the operator may since have changed. - network.netd_interface = if filtered { - NetdInterface::Filtered - } else { - NetdInterface::Unfiltered - }; + prepared.push(identity.clone()); // Everything below runs after netd already built a host interface, // so a failure has to unwind the same way a failed Prepare does. let accepted = (|| { @@ -851,13 +828,6 @@ impl App { network.nic.vhost = Some(false); network.nic.queues = Some(1); } - for network in &mut networks { - network.netd_interface = match netd_teardown(network, &self.config.cvm) { - Some(true) => NetdInterface::Filtered, - Some(false) => NetdInterface::Unfiltered, - None => NetdInterface::None, - }; - } networks } @@ -870,13 +840,10 @@ impl App { } /// Removes interfaces netd already built for a launch that then failed. - async fn roll_back_prepared_networks(&self, prepared: Vec<(InterfaceIdentity, bool)>) { - for (identity, filtered) in prepared.into_iter().rev() { - if let Err(cleanup_error) = netd::request( - &self.config.netd.socket, - &NetdRequest::Remove { identity, filtered }, - ) - .await + async fn roll_back_prepared_networks(&self, prepared: Vec) { + for identity in prepared.into_iter().rev() { + if let Err(cleanup_error) = + netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity }).await { warn!(%cleanup_error, "failed to roll back prepared network interface"); } diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index 05fb6e42e..e9145d3dd 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -11,8 +11,7 @@ use sha2::{Digest, Sha256}; use super::{Manifest, PortMapping}; use crate::config::{ - CvmConfig, NetdInterface, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, - MAX_NET_QUEUES, + CvmConfig, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, MAX_NET_QUEUES, }; /// Node configuration merged with what one NIC pins. @@ -39,7 +38,6 @@ pub(crate) fn resolve_networking( // Runtime state, never inherited from configuration or from a previous // launch. Interface preparation sets both for the NICs it builds, and a // node configuration that names either is rejected at startup. - resolved.netd_interface = crate::config::NetdInterface::None; resolved.device.clear(); if !networking.bridge.is_empty() { resolved.nic.bridge = networking.bridge.clone(); @@ -110,28 +108,6 @@ pub(crate) fn filters_bridge_traffic(networking: &Networking, cfg: &CvmConfig) - && cfg.network_filter.mode == NetworkFilterMode::Libvirt } -/// Whether netd built this NIC's host interface, and if so whether it carries -/// an nwfilter binding. -/// -/// Interface preparation records this, because it is not derivable afterwards: -/// an operator can change `network_filter.mode` or `max_net_queues` while a VM -/// runs, and teardown has to undo what was built rather than what would be -/// built now. -pub(crate) fn netd_teardown(networking: &Networking, cfg: &CvmConfig) -> Option { - match networking.netd_interface { - NetdInterface::Filtered => Some(true), - NetdInterface::Unfiltered => Some(false), - // Either nothing was built, or this entry was persisted before - // preparation recorded the fact. Fall back to the derivation such an - // entry was created by; a Remove for an interface that does not exist - // is a no-op. - NetdInterface::None if needs_netd_interface(networking) => { - Some(filters_bridge_traffic(networking, cfg)) - } - NetdInterface::None => None, - } -} - /// Makes the data plane concrete on a launch-time NIC list. /// /// `vhost` on a freshly resolved entry is still a *request*: `None` means @@ -228,30 +204,23 @@ pub(crate) fn warn_if_vhost_net_missing(networks: &[Networking]) { /// Which NIC an unpinned port mapping's traffic enters through. /// -/// The first user-mode NIC, because that is where QEMU's `hostfwd=` entries -/// have always gone and existing VMs must keep behaving the same way; failing -/// that the first bridge NIC, which is the only other backend with a path into -/// the guest. `macvtap` bypasses the host bridge and `custom` owns its own -/// netdev string, so neither can carry one. +/// The first user-mode NIC, which is where QEMU's `hostfwd=` entries have +/// always gone. There is no second choice: nothing else on this host publishes +/// a port, so a VM without one has nowhere to put a mapping and the launch +/// says so. pub(crate) fn default_ingress_nic(networks: &[Networking]) -> Option { networks .iter() .position(|network| network.nic.mode == NetworkingMode::User) - .or_else(|| { - networks - .iter() - .position(|network| network.nic.mode == NetworkingMode::Bridge) - }) } /// Whether a NIC of this mode has a mechanism to publish a host port at all. /// -/// `hostfwd=` for user mode and netd for a bridge. Macvtap bypasses the host -/// bridge and a custom netdev is a string the VMM does not interpret, so -/// neither has anywhere to put one. Naming one of those is refused at -/// deployment rather than resolved to nothing here. +/// QEMU's `hostfwd=`, and nothing else. A bridge TAP is built by netd, and the +/// netd in this repository does not forward host ports; macvtap bypasses the +/// host bridge; a custom netdev is a string the VMM does not interpret. pub(crate) fn mode_carries_ingress(mode: NetworkingMode) -> bool { - matches!(mode, NetworkingMode::User | NetworkingMode::Bridge) + matches!(mode, NetworkingMode::User) } /// Which NIC a port mapping's traffic enters through. @@ -260,10 +229,9 @@ pub(crate) fn mode_carries_ingress(mode: NetworkingMode) -> bool { /// mechanism: `hostfwd=` for user mode, netd for a bridge. That is what keeps /// QEMU and netd from both claiming one host port. /// -/// `None` is a mapping with nowhere to go. Deployment refuses every way of -/// asking for one, so reaching it means a manifest wrote a NIC out from under a -/// mapping that named it; the launch says so rather than dropping it in -/// silence. +/// `None` is a mapping with nowhere to go: a VM with no user-mode NIC, or one +/// whose NICs changed under a mapping that named one. The launch warns about +/// each of those rather than dropping it in silence. pub(crate) fn ingress_nic(mapping: &PortMapping, networks: &[Networking]) -> Option { mapping .nic_index @@ -285,24 +253,6 @@ pub(crate) fn stranded_ingress<'a>( .filter(|mapping| ingress_nic(mapping, networks).is_none()) } -/// The host ports one NIC carries, as netd requests. -pub(crate) fn ingress_for( - port_map: &[PortMapping], - networks: &[Networking], - nic_index: usize, -) -> Vec { - port_map - .iter() - .filter(|mapping| ingress_nic(mapping, networks) == Some(nic_index)) - .map(|mapping| crate::netd::IngressRequest { - protocol: mapping.protocol.as_str().to_string(), - host_address: mapping.address.to_string(), - host_port: mapping.from, - guest_port: mapping.to, - }) - .collect() -} - /// Derives a deterministic, locally administered unicast MAC address. /// /// Index zero preserves the legacy single-NIC derivation. Later interfaces @@ -330,9 +280,8 @@ pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) #[cfg(test)] mod tests { use super::{ - default_ingress_nic, ingress_for, ingress_nic, mac_address_for_vm_index, - needs_netd_interface, netd_teardown, resolve_networking, resolved_networks, settle_vhost, - stranded_ingress, validate_resolved_networks, + default_ingress_nic, ingress_nic, mac_address_for_vm_index, needs_netd_interface, + resolved_networks, settle_vhost, stranded_ingress, validate_resolved_networks, }; use crate::app::PortMapping; use crate::config::Protocol; @@ -566,77 +515,6 @@ mod tests { assert_eq!(unset[0].nic.vhost, Some(false)); } - /// Teardown has to undo what was built. Node configuration is mutable and - /// a VM outlives an edit to it, so re-deriving "did netd build this?" at - /// removal time orphans TAPs and leaks nwfilter bindings whose ebtables - /// rules the next VM at the same deterministic interface name inherits. - #[test] - fn teardown_follows_what_was_built_not_what_configuration_now_says() { - use crate::config::{NetdInterface, NetworkFilterMode}; - - let filtering = { - let mut cvm = node_config(NetworkingMode::Bridge); - cvm.network_filter.mode = NetworkFilterMode::Libvirt; - cvm - }; - let unfiltered = node_config(NetworkingMode::Bridge); - - let mut built_filtered = filtering.networking.clone(); - built_filtered.nic.queues = Some(1); - built_filtered.netd_interface = NetdInterface::Filtered; - // The operator turns filtering off while the VM runs. The binding is - // still there and still has to be deleted. - assert_eq!(netd_teardown(&built_filtered, &unfiltered), Some(true)); - - let mut built_unfiltered = unfiltered.networking.clone(); - built_unfiltered.nic.queues = Some(4); - built_unfiltered.netd_interface = NetdInterface::Unfiltered; - // The operator turns filtering on. There is no binding to delete, and - // asking libvirt for one would fail the removal. - assert_eq!(netd_teardown(&built_unfiltered, &filtering), Some(false)); - - // A backend netd never builds stays untouched, whatever the node says. - let mut untouched = unfiltered.networking.clone(); - untouched.nic.mode = NetworkingMode::User; - untouched.nic.queues = Some(1); - assert_eq!(netd_teardown(&untouched, &unfiltered), None); - - // A bridge NIC with no record is netd's by derivation, because netd is - // now the only thing that could have built it. Teardown deletes by - // deriving names, so being wrong about a VM from an older build costs - // a sweep that finds nothing. - let mut unrecorded = unfiltered.networking.clone(); - unrecorded.nic.queues = Some(1); - assert_eq!(unrecorded.netd_interface, NetdInterface::None); - assert_eq!(netd_teardown(&unrecorded, &unfiltered), Some(false)); - - // An entry persisted before preparation recorded the fact still gets - // torn down by the rule that created it. - let mut legacy = filtering.networking.clone(); - legacy.nic.queues = Some(1); - assert_eq!(legacy.netd_interface, NetdInterface::None); - assert_eq!(netd_teardown(&legacy, &filtering), Some(true)); - } - - /// Resolution produces launch input, never a claim about what exists. - #[test] - fn resolution_never_carries_a_stale_interface_record() { - use crate::config::NetdInterface; - - let cvm = node_config(NetworkingMode::Bridge); - // The node does not filter, so only a stale record could make teardown - // ask libvirt to delete a binding. - let mut previous = cvm.networking.clone(); - previous.nic.queues = Some(1); - previous.netd_interface = NetdInterface::Filtered; - assert_eq!(netd_teardown(&previous, &cvm), Some(true)); - - let resolved = resolve_networking(&previous.nic, &cvm, 4); - assert_eq!(resolved.netd_interface, NetdInterface::None); - // Back to the derivation the node's own configuration gives. - assert_eq!(netd_teardown(&resolved, &cvm), Some(false)); - } - #[test] fn primary_mac_keeps_legacy_derivation_and_later_nics_are_distinct() { assert_eq!( @@ -677,12 +555,12 @@ mod tests { assert_eq!(default_ingress_nic(&networks), Some(1)); assert_eq!(ingress_nic(&mapping(443, None), &networks), Some(1)); - // With no user-mode NIC there was nowhere at all, which is the hole - // this closes: a bridge NIC is the only other backend with a path. + // With no user-mode NIC there is nowhere at all. netd builds a bridge + // TAP but does not forward host ports, and macvtap and custom have no + // path either. let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::Bridge)]; - assert_eq!(default_ingress_nic(&networks), Some(0)); + assert_eq!(default_ingress_nic(&networks), None); - // macvtap bypasses the host bridge and custom owns its netdev string. let networks = [nic(NetworkingMode::Macvtap), nic(NetworkingMode::Custom)]; assert_eq!(default_ingress_nic(&networks), None); assert_eq!(ingress_nic(&mapping(443, None), &networks), None); @@ -691,7 +569,7 @@ mod tests { #[test] fn a_pinned_mapping_goes_where_it_says() { let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; - assert_eq!(ingress_nic(&mapping(443, Some(0)), &networks), Some(0)); + assert_eq!(ingress_nic(&mapping(443, Some(1)), &networks), Some(1)); // Out of range resolves to nothing rather than to something arbitrary. // Deployment refuses it outright; a manifest that lost a NIC lands here. assert_eq!(ingress_nic(&mapping(443, Some(7)), &networks), None); @@ -707,10 +585,13 @@ mod tests { nic(NetworkingMode::Macvtap), nic(NetworkingMode::Custom), nic(NetworkingMode::Bridge), + nic(NetworkingMode::User), ]; assert_eq!(ingress_nic(&mapping(443, Some(0)), &networks), None); assert_eq!(ingress_nic(&mapping(443, Some(1)), &networks), None); - assert_eq!(ingress_nic(&mapping(443, Some(2)), &networks), Some(2)); + // A bridge TAP is netd's, and netd does not forward host ports. + assert_eq!(ingress_nic(&mapping(443, Some(2)), &networks), None); + assert_eq!(ingress_nic(&mapping(443, Some(3)), &networks), Some(3)); // And an unpinned mapping on a VM with nowhere to put it is named, // rather than counted as delivered. @@ -721,26 +602,4 @@ mod tests { .collect(); assert_eq!(stranded, vec![443, 8080]); } - - #[test] - fn one_mapping_reaches_exactly_one_nic() { - // The property that keeps QEMU and netd from both claiming a host port: - // every mapping appears under one NIC and no other. - let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; - let port_map = [ - mapping(443, Some(0)), - mapping(8080, None), - mapping(9090, Some(1)), - ]; - let per_nic: Vec<_> = (0..networks.len()) - .map(|index| ingress_for(&port_map, &networks, index)) - .collect(); - // Only NIC 0 is a bridge, so only its list becomes netd requests; the - // other two ride QEMU's hostfwd on NIC 1. - assert_eq!(per_nic[0].len(), 1); - assert_eq!(per_nic[0][0].host_port, 443); - assert_eq!(per_nic[1].len(), 2); - let total: usize = per_nic.iter().map(Vec::len).sum(); - assert_eq!(total, port_map.len()); - } } diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 52fb22a24..fe420a484 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -683,10 +683,10 @@ impl QemuCommandBuilder<'_> { ); // netd owns the TAP. It is the one component here with // CAP_NET_ADMIN, so it is the only one that can bind an - // nwfilter, create a persistent IFF_MULTI_QUEUE device, or - // arbitrate a host port between VMM instances -- and having - // it own every bridge TAP is what keeps a VM's networking - // from depending on which of those a node happens to use. + // nwfilter or create a persistent IFF_MULTI_QUEUE device -- + // and having it own every bridge TAP is what keeps a VM's + // networking from depending on which of those a node + // happens to use. let tap = tap_name(&InterfaceIdentity { instance_id: self.cfg.instance_id.clone(), vm_id: self.vm.manifest.id.clone(), diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index e6b55ae6f..834dc0673 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -906,10 +906,6 @@ fn validate_networking(networking: &Networking) -> Result<()> { !networking.nic.inherit_mode, "cvm.networking.inherit_mode is per-deployment state and cannot be set on the node default" ); - anyhow::ensure!( - networking.netd_interface.is_none(), - "cvm.networking.netd_interface is runtime state and cannot be set in configuration" - ); anyhow::ensure!( networking.device.is_empty(), "cvm.networking.device is runtime state and cannot be set in configuration" @@ -1075,41 +1071,7 @@ pub struct Networking { // ── Custom fields ────────────────────────────────────────────── #[serde(default)] pub netdev: String, - // ── Runtime markers ──────────────────────────────────────────── - /// What netd built for this NIC, recorded when it was built. - /// - /// Runtime state, like `device`: resolution always clears it. Teardown - /// reads this rather than re-deriving it from node configuration, because - /// an operator may change `network_filter.mode` or `max_net_queues` while - /// the VM runs, and what has to be removed is what was created. - #[serde(default, skip_serializing_if = "NetdInterface::is_none")] - pub netd_interface: NetdInterface, -} - -/// The host interface netd created for a NIC, if any. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum NetdInterface { - /// netd was not involved: user mode, custom mode, or a bridge NIC that - /// QEMU's own bridge helper attaches. - #[default] - None, - /// netd created the interface and bound no libvirt nwfilter to it. - Unfiltered, - /// netd created the interface and bound a libvirt nwfilter to it, which - /// removal has to delete before the interface goes away. - Filtered, -} - -impl NetdInterface { - pub fn is_none(&self) -> bool { - matches!(self, NetdInterface::None) - } - - pub fn is_filtered(&self) -> bool { - matches!(self, NetdInterface::Filtered) - } } impl Networking { @@ -1462,25 +1424,21 @@ mod tests { .expect("default VMM config should parse") } - /// The two ownership markers are additive on disk: manifests and runtime - /// network snapshots written before they existed still load, and an entry - /// that carries neither serializes exactly as it used to. + /// The ownership marker is additive on disk: manifests and runtime network + /// snapshots written before it existed still load, and an entry that does + /// not carry it serializes exactly as it used to. #[test] fn ownership_markers_are_omitted_when_unset_and_default_when_absent() { let mut networking: Networking = serde_json::from_str(r#"{"mode":"bridge","bridge":"br0"}"#).unwrap(); assert!(!networking.nic.inherit_mode); - assert_eq!(networking.netd_interface, NetdInterface::None); let json = serde_json::to_string(&networking).unwrap(); assert!(!json.contains("inherit_mode"), "{json}"); - assert!(!json.contains("netd_interface"), "{json}"); networking.nic.inherit_mode = true; - networking.netd_interface = NetdInterface::Filtered; let json = serde_json::to_string(&networking).unwrap(); assert!(json.contains(r#""inherit_mode":true"#), "{json}"); - assert!(json.contains(r#""netd_interface":"filtered""#), "{json}"); assert_eq!( serde_json::from_str::(&json).unwrap(), networking @@ -1678,7 +1636,6 @@ mod networking_shape_tests { "vhost": false, "queues": 4, "inherit_mode": true, - "netd_interface": "filtered", }); // A resolved value keeps every field, at the same names as before. diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index f1a72c59f..cbf3e9687 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -199,22 +199,8 @@ fn resolved_nic_modes( /// warning either. fn validate_port_mapping_nics(mappings: &[PortMapping], modes: &[NetworkingMode]) -> Result<()> { let nic_count = modes.len(); - let has_ingress = modes.iter().any(|mode| mode_carries_ingress(*mode)); for mapping in mappings { let Some(index) = mapping.nic_index else { - // Unpinned, so it goes wherever the default resolves -- unless - // there is nowhere for it to resolve to. That is a property of the - // VM's own topology, which is why it is decided here and not by - // asking the host anything. - if !has_ingress { - bail!( - "port mapping {} {}:{} has no NIC to enter through: this VM has no user-mode \ - or bridge interface", - mapping.protocol.as_str(), - mapping.address, - mapping.from - ); - } continue; }; let Some(mode) = modes.get(index) else { @@ -1490,7 +1476,10 @@ mod tests { ]; let modes = resolved_nic_modes(&bridge_then_macvtap, &cvm, 2); assert_eq!(modes, vec![NetworkingMode::Bridge, NetworkingMode::Macvtap]); - validate_port_mapping_nics(&[pinned(Some(0))], &modes).unwrap(); + + // A bridge TAP is netd's, and netd does not forward host ports. + let error = validate_port_mapping_nics(&[pinned(Some(0))], &modes).unwrap_err(); + assert!(error.to_string().contains("bridge"), "{error}"); let error = validate_port_mapping_nics(&[pinned(Some(1))], &modes).unwrap_err(); assert!(error.to_string().contains("macvtap"), "{error}"); diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 6422aec25..dd8a39671 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -105,50 +105,6 @@ pub struct PrepareBridgeRequest { /// it without going through the VMM. #[serde(default)] pub workdir: String, - /// Host ports this VM wants reachable at its guest. - /// - /// Empty asks for nothing, which is also what a caller predating the field - /// sends. Whether a netd forwards them is its own business; this states the - /// requirement rather than assuming it is met, and the response says what - /// was actually done. - /// - /// Owned by the interface. Whatever a netd establishes to satisfy this is - /// released when the interface is -- by `remove`, by `remove_all`, or by a - /// collection -- and there is deliberately no operation that releases it - /// separately. A host port outliving the interface it was forwarding to is - /// a leak nothing would ever collect: the interface is the only thing that - /// carries an ownership record, so a reservation that survived it could - /// never be attributed to a VM again. A prepare for an identity that - /// already has an interface replaces both together. - #[serde(default)] - pub ingress: Vec, -} - -/// One host port a VM wants reachable at its guest. -/// -/// Every field is named by the caller, which is what `bridge`, `mac` and -/// `queues` already get and the opposite of `filtered`. The difference is -/// whether netd can check what it is handed: an nwfilter name cannot be checked -/// for whether it filters anything, while a host port is a closed space a node -/// policy can be stated over. Naming is not deciding -- which ports may be -/// handed out stays netd's own configuration, exactly as `allowed_bridges` -/// governs the bridge a caller names. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct IngressRequest { - /// `"tcp"` or `"udp"`. - pub protocol: String, - /// Host address to accept on. Empty leaves the choice to netd. - /// - /// Not decoration: an admin port bound to loopback and a published one - /// differ only here. - #[serde(default)] - pub host_address: String, - /// Host port, as the deployment named it. There is no "pick one for me": - /// the caller reports this number back through `GetInfo` and a client - /// connects to it, so a netd-chosen port would have to travel back through - /// both before it meant anything. - pub host_port: u16, - pub guest_port: u16, } /// One host resource netd holds. @@ -204,15 +160,6 @@ pub enum Request { Remove { #[serde(flatten)] identity: InterfaceIdentity, - /// Whether this interface was created with an nwfilter binding. - /// - /// Advisory, and no longer read: removal detects macvtap itself, and - /// clears a binding best-effort whatever this says. The strict rule it - /// used to select exists for prepare, where a binding left at the name - /// blocks the one about to be created; at removal it only left the - /// interface up on the bridge. Still sent, and still required on - /// decode, so a netd that predates that reasoning keeps working. - filtered: bool, }, /// Delete every interface netd holds for one VM. /// @@ -222,9 +169,6 @@ pub enum Request { /// that lost a NIC leaves the same thing behind. Both are found here /// without a record, because every name netd can produce for a VM is /// derivable from its identity. - /// - /// Releases everything those interfaces own, host ports included. See - /// [`PrepareBridgeRequest::ingress`]. RemoveAll { instance_id: String, vm_id: String, @@ -794,10 +738,7 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; Ok(Outcome::tap(tap)) } - Request::Remove { - identity, - filtered: _, - } => { + Request::Remove { identity } => { validate_identity(&identity)?; let tap = tap_name(&identity); // Best effort, whatever the caller says was built. The strict rule @@ -1735,7 +1676,6 @@ mod tests { filtered: true, queues: 0, workdir: String::new(), - ingress: Vec::new(), }; let filter = NetworkFilterConfig { mode: crate::config::NetworkFilterMode::Libvirt, @@ -1763,7 +1703,6 @@ mod tests { fn remove_protocol_keeps_identity_fields_flat() { let request = Request::Remove { identity: identity("instance", "vm", 2), - filtered: true, }; let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "remove"); @@ -1783,7 +1722,6 @@ mod tests { filtered: true, queues: 0, workdir: String::new(), - ingress: Vec::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -1792,40 +1730,6 @@ mod tests { assert!(value.get("identity").is_none()); } - /// `filtered` says which of two shapes was built, and both are reachable - /// on any node this build can produce. There is no released peer that omits - /// it -- netd does not exist before v0.6 -- so it is required rather than - /// defaulted, and a request that leaves it out is a bug, not an old client. - #[test] - fn removal_states_which_shape_it_is_undoing() { - let error = serde_json::from_value::(serde_json::json!({ - "operation": "remove", - "instance_id": "instance", - "vm_id": "vm", - "nic_index": 0, - })) - .unwrap_err(); - assert!(error.to_string().contains("filtered"), "{error}"); - - for filtered in [true, false] { - let decoded: Request = serde_json::from_value(serde_json::json!({ - "operation": "remove", - "instance_id": "instance", - "vm_id": "vm", - "nic_index": 0, - "filtered": filtered, - })) - .unwrap(); - let Request::Remove { - filtered: decoded, .. - } = decoded - else { - panic!("wrong variant"); - }; - assert_eq!(decoded, filtered); - } - } - /// A binding outlives the interface it was bound to, and TAP names are a /// deterministic hash of the VM identity, so the same name comes back. /// Removing an interface therefore clears whatever binding is there, and @@ -1854,7 +1758,6 @@ mod tests { filtered: false, queues: 4, workdir: String::new(), - ingress: Vec::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["queues"], 4); @@ -1906,7 +1809,6 @@ mod tests { filtered: true, queues: 0, workdir: String::new(), - ingress: Vec::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -1984,7 +1886,6 @@ mod tests { filtered: false, queues: 4, workdir: String::new(), - ingress: Vec::new(), }; let filtering = NetworkFilterConfig { mode: crate::config::NetworkFilterMode::Libvirt, @@ -2058,7 +1959,6 @@ mod tests { filtered: true, queues: 1, workdir: String::new(), - ingress: Vec::new(), }; // Nothing on the wire can name a filter: the field does not exist. let wire = serde_json::to_value(Request::PrepareBridge(request.clone())).unwrap(); @@ -2099,7 +1999,6 @@ mod tests { filtered: true, queues: 1, workdir: "/opt/dstack/run/vm/vm".into(), - ingress: Vec::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["workdir"], "/opt/dstack/run/vm/vm"); @@ -2111,37 +2010,6 @@ mod tests { assert_eq!(decoded.workdir, ""); } - #[test] - fn host_ports_travel_with_the_bridge_prepare_and_default_to_none() { - let request = Request::PrepareBridge(PrepareBridgeRequest { - identity: identity("instance", "vm", 0), - bridge: "br0".into(), - mac: "02:00:00:00:00:01".into(), - qemu_uid: 1000, - filtered: true, - queues: 1, - workdir: String::new(), - ingress: vec![IngressRequest { - protocol: "udp".into(), - host_address: "0.0.0.0".into(), - host_port: 7483, - guest_port: 51820, - }], - }); - let value = serde_json::to_value(request).unwrap(); - assert_eq!(value["ingress"][0]["protocol"], "udp"); - assert_eq!(value["ingress"][0]["host_port"], 7483); - assert_eq!(value["ingress"][0]["guest_port"], 51820); - // The bind address separates an admin port from a published one, so a - // forwarder that lost it would publish the admin port. - assert_eq!(value["ingress"][0]["host_address"], "0.0.0.0"); - - let Request::PrepareBridge(decoded) = decode_minimal_bridge() else { - panic!("expected a bridge prepare"); - }; - assert!(decoded.ingress.is_empty()); - } - /// A prepare carrying only the fields that predate this change. fn decode_minimal_bridge() -> Request { serde_json::from_value(serde_json::json!({ diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 11354a055..6be8400aa 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -150,9 +150,9 @@ parameters = {} # Shared privileged networking service. Required by bridge and macvtap # networking: it builds every host interface those modes use, binds their -# nwfilters, and arbitrates host ports between VMM instances. User mode and a -# caller-supplied netdev need nothing from it. Socket filesystem permissions -# authorize clients. +# nwfilters, and releases them again. It does not forward host ports. User mode +# and a caller-supplied netdev need nothing from it. Socket filesystem +# permissions authorize clients. [netd] socket = "/run/dstack/netd.sock" # How often the VMM asks netd to collect interfaces no VM of its claims. The From d35ade3efed1a1089ce73cc3652e57f2895359ee Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 08:24:48 -0700 Subject: [PATCH 30/34] refactor(vmm): one notion of a failed sweep, not two and a bit `sweep_vm_interfaces` reported failure three ways. A binding it could not delete was an error in the branch for an orphaned one and a `warn!` plus an `incomplete` bit in the branch for a live one -- the same failure, read two ways, and the silent reading was the one that mattered. Both halves now fold into the same `first_error`, which folds the two branches into one loop and retires `remove_interface_in_pass`. That leaves `incomplete` with one producer: `COLLECTION_DEADLINE`, which cannot fire. The pass is 256 `stat()` calls; libvirt is asked once for a listing and at most once more for a delete, because the first failure latches it off. To spend twenty seconds it would need ~250 interfaces present for a single VM, against a `validate_identity` cap of 256 NICs and a real VM's one to four. The bit was threaded through `Outcome::Swept`, `Response`, `netd::Sweep`, a `warn!` and a `println!`, and no branch anywhere read it. `remove_all` returns a count. `InterfaceRecord.bound` goes the same way: written by `list_interfaces`, read only by the `FILTERED` column of `netd list`, where `kind` already says `binding` for the one case it distinguishes -- and it reported "no" both for an interface with no binding and for one libvirt could not be asked about. `create_vm` validated its port mapping NICs twice, on the same mappings against the same modes, because `create_manifest_from_vm_config` had already done it. Deleting the second call lets the UI drop `originalPorts` and the `JSON.stringify` deep-compare it needed to decide whether to send `update_ports`: a read-modify-write client does send the ports, so saying so is honest, and what the server checks on that path is now a local question about the VM's own pins. Also inlines `binding_cleanup` into its only caller, and corrects three comments the capability-probe removal left describing things the code no longer does. --- dstack/vmm/src/app.rs | 29 +-- dstack/vmm/src/main.rs | 12 +- dstack/vmm/src/main_service.rs | 4 - dstack/vmm/src/netd.rs | 198 +++++------------- dstack/vmm/ui/src/composables/useVmManager.ts | 17 +- 5 files changed, 60 insertions(+), 200 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 2f629b469..7b8484664 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1049,25 +1049,9 @@ impl App { ) .await { - Ok(sweep) => { - if sweep.removed > 0 { - info!( - vm_id, - removed = sweep.removed, - "released netd-managed interfaces" - ); - } - // A sweep that ran out of time is not a sweep that found - // nothing left. Saying so is the difference between a host an - // operator can reason about and one where "released 3 - // interfaces" hid the fourth. - if sweep.incomplete { - warn!( - vm_id, - "netd stopped releasing this VM's interfaces on its deadline; the rest \ - go when this VM next launches, or -- once it is removed and nothing \ - claims it -- when reconciliation next runs" - ); + Ok(removed) => { + if removed > 0 { + info!(vm_id, removed, "released netd-managed interfaces"); } } Err(error) if netd::is_unreachable(&error) => { @@ -1613,9 +1597,9 @@ impl App { pub async fn vm_info(&self, id: &str) -> Result> { let proc_state = self.supervisor.info(id).await?; - // Snapshot under the lock, then release it: describing the VM can - // probe netd, and that is a blocking connect the global state lock has - // no business being held across. + // Snapshot under the lock, then release it: the global state lock is + // held by every other VM's operations, and describing one VM has no + // business keeping it across the work that follows. let info = { let state = self.lock(); let Some(vm_state) = state.get(id) else { @@ -2439,7 +2423,6 @@ mod tests { "instance_id": instance, "vm_id": vm, "nic_index": 0, - "bound": false, }) } diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 627c2fc18..1985a2a39 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -243,7 +243,7 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< .await .context("failed to list netd interfaces")?; println!( - "{:<16} {:<8} {:<24} {:<38} {:>3} FILTERED", + "{:<16} {:<8} {:<24} {:<38} {:>3}", "INTERFACE", "KIND", "INSTANCE", "VM", "NIC" ); let mut unattributed = 0; @@ -252,7 +252,7 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< unattributed += 1; } println!( - "{:<16} {:<8} {:<24} {:<38} {:>3} {}", + "{:<16} {:<8} {:<24} {:<38} {:>3}", record.tap, record.kind, record.instance_id.as_deref().unwrap_or("-"), @@ -260,7 +260,6 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< record .nic_index .map_or_else(|| "-".to_string(), |index| index.to_string()), - if record.bound { "yes" } else { "no" }, ); } println!(); @@ -283,13 +282,10 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< Ok(()) } NetdCommand::RemoveVm { instance, vm } => { - let sweep = netd::remove_all(&config.socket, instance, vm) + let removed = netd::remove_all(&config.socket, instance, vm) .await .context("failed to remove the VM's interfaces")?; - println!("removed {} interface(s) for {vm}", sweep.removed); - if sweep.incomplete { - println!("netd stopped on its deadline; run this again to continue"); - } + println!("removed {removed} interface(s) for {vm}"); Ok(()) } } diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index cbf3e9687..1c2835942 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -837,10 +837,6 @@ impl VmmRpc for RpcHandler { async fn create_vm(self, request: VmConfiguration) -> Result { let manifest = create_manifest_from_vm_config(request.clone(), &self.app.config.cvm)?; self.validate_port_mapping_conflicts(None, &manifest.port_map)?; - validate_port_mapping_nics( - &manifest.port_map, - &resolved_nic_modes(&manifest.networks, &self.app.config.cvm, manifest.vcpu), - )?; let id = manifest.id.clone(); info!(vm_id = %id, "create_vm RPC called"); let app_id = manifest.app_id.clone(); diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index dd8a39671..0f8e07d6c 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -58,20 +58,6 @@ const TAP_DIGEST_CHARS: usize = 12; const ALIAS_PREFIX: &str = "dstack1"; /// What the kernel stores in an interface alias, minus the terminator. const MAX_IFALIAS: usize = 255; -/// How long a sweep or collection keeps starting work on new interfaces. -/// -/// A bound on when the pass stops, not on how long one interface takes: an -/// interface it has already begun is still bounded only by `COMMAND_TIMEOUT` -/// per helper invocation, so a pass that passes this check at 19.9s can run on -/// for as long as the `ip` and `virsh` calls it has started take to time out. -/// -/// What it protects is the caller's patience against the operation lock. Every -/// other prepare and remove on the host waits behind a pass, and the caller -/// that asked for this one gave up at thirty seconds; work done after that is -/// work nobody is waiting for, done while everybody waits. Partial progress -/// reported honestly beats total progress reported to nobody. -const COLLECTION_DEADLINE: Duration = Duration::from_secs(20); - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InterfaceIdentity { pub instance_id: String, @@ -128,9 +114,6 @@ pub struct InterfaceRecord { pub vm_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub nic_index: Option, - /// Whether libvirt holds an nwfilter binding at this name. - #[serde(default)] - pub bound: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -226,10 +209,6 @@ struct Response { /// How many interfaces a whole-VM sweep deleted. #[serde(default, skip_serializing_if = "Option::is_none")] removed: Option, - /// Whether the pass stopped on its deadline with work left. See - /// [`COLLECTION_DEADLINE`]. - #[serde(default, skip_serializing_if = "Option::is_none")] - incomplete: Option, /// For a listing, everything netd holds; for a collection, what it took, /// or would take on a dry run. Absent, rather than empty, from a netd that /// cannot enumerate: "I hold nothing" and "I cannot say" are answers a @@ -243,7 +222,7 @@ struct Response { /// What one request produced. /// /// An enum rather than a struct of options, because the shapes do not overlap: -/// a prepare names an interface, a sweep counts them, a probe names neither. +/// a prepare names an interface, a sweep counts them, a listing enumerates. /// One flat [`Response`] still carries all of them on the wire, so a netd that /// grows an operation stays readable to a caller that does not know it. enum Outcome { @@ -256,7 +235,6 @@ enum Outcome { /// A sweep names no single interface, so it reports how many it deleted. Swept { removed: usize, - incomplete: bool, }, Listed(Vec), } @@ -277,7 +255,6 @@ impl Outcome { device: None, queues: None, removed: None, - incomplete: None, interfaces: None, error: None, }; @@ -291,13 +268,7 @@ impl Outcome { response.device = device; response.queues = queues; } - Self::Swept { - removed, - incomplete, - } => { - response.removed = Some(removed); - response.incomplete = Some(incomplete); - } + Self::Swept { removed } => response.removed = Some(removed), Self::Listed(interfaces) => response.interfaces = Some(interfaces), } response @@ -446,31 +417,15 @@ pub async fn request(socket: &Path, request: &Request) -> Result Result { +pub async fn remove_all(socket: &Path, instance_id: &str, vm_id: &str) -> Result { let request = Request::RemoveAll { instance_id: instance_id.to_string(), vm_id: vm_id.to_string(), }; - let response = exchange(socket, &request).await?; - Ok(Sweep { - removed: response - .removed - .context("netd answered a sweep without saying what it removed")?, - // Absent from a netd that cannot stop early, which is the same as one - // that did not. - incomplete: response.incomplete.unwrap_or_default(), - }) -} - -/// What one whole-VM sweep did. -#[derive(Debug, Clone, Copy)] -pub struct Sweep { - pub removed: usize, - /// Whether it stopped on its deadline with names left to check. Reported - /// rather than dropped: a sweep that ran out of time and one that finished - /// having removed the same count are different states of the host, and - /// only one of them needs looking at. - pub incomplete: bool, + exchange(socket, &request) + .await? + .removed + .context("netd answered a sweep without saying what it removed") } /// Deletes one interface by name. See [`Request::RemoveInterface`]. @@ -641,10 +596,10 @@ async fn serve_connection( .await .context("timed out reading a netd request")?; let outcome = match request { - // A peer that connects and closes without sending is the VMM's - // reachability check: netd that died leaves its socket behind, so the - // VMM connects to tell the two apart. Answering that with a parse error - // and a warning would fill the log with reports of it working. + // A peer that connects and closes without sending is asking whether + // anything is listening: netd that died leaves its socket behind. + // Answering that with a parse error and a warning would fill the log + // with reports of it working. Ok(None) => { debug!("netd liveness probe"); return Ok(()); @@ -677,7 +632,6 @@ async fn serve_connection( device: None, queues: None, removed: None, - incomplete: None, interfaces: None, error: Some(format!("{error:#}")), } @@ -725,11 +679,8 @@ fn handle_request(config: &NetdConfig, request: Request) -> Result { Ok(Outcome::Listed(list_interfaces(libvirt_uri, &instance_id))) } Request::RemoveAll { instance_id, vm_id } => { - let (removed, incomplete) = sweep_vm_interfaces(libvirt_uri, &instance_id, &vm_id)?; - Ok(Outcome::Swept { - removed, - incomplete, - }) + let removed = sweep_vm_interfaces(libvirt_uri, &instance_id, &vm_id)?; + Ok(Outcome::Swept { removed }) } Request::RemoveInterface { tap } => { if !is_managed_name(&tap) { @@ -893,7 +844,19 @@ fn prepare_bridge( let tap = tap_name(&request.identity); // A failed VMM start may leave a deterministic resource behind. Replacing // it makes prepare idempotent without accepting a caller-selected TAP. - remove_interface(libvirt_uri, &tap, binding_cleanup(filtered))?; + // A binding outlives the interface it was bound to and TAP names are + // derived, so the same name comes back: clear whatever is there. Insist + // only when this prepare is about to create a replacement libvirt would + // refuse as a duplicate. + remove_interface( + libvirt_uri, + &tap, + if filtered { + BindingCleanup::Required + } else { + BindingCleanup::BestEffort + }, + )?; let uid = request.qemu_uid.to_string(); let queues = validate_queues(request.queues)?; @@ -955,29 +918,6 @@ enum BindingCleanup { Skip, } -/// Removes one interface during a pass over many. -/// -/// A pass gets one answer about libvirt, not one per interface. Asking again -/// after it has failed is how a hung `libvirtd` turns a bounded collection into -/// an unbounded one: the deadline is reached having deleted nothing, while -/// every other VM on the host waits behind the operation lock. -/// `Ok(false)` when the interface is gone but its binding could not be -/// cleared. The pass reports that as incomplete rather than as success: the -/// orphan half of the same loop already treats a binding it could not delete -/// as a failure, and a stale binding is exactly the state a later filtered -/// prepare at the same name hard-fails on. -fn remove_interface_in_pass(uri: &str, tap: &str, libvirt: &mut bool) -> Result { - let mut cleared = true; - if *libvirt && !is_macvtap(tap) { - if let Err(error) = delete_binding(uri, tap) { - warn!(%tap, "could not clear a possible nwfilter binding: {error:#}"); - *libvirt = false; - cleared = false; - } - } - remove_interface(uri, tap, BindingCleanup::Skip).map(|()| cleared) -} - /// Deletes every interface a VM could hold, by deriving each name rather than /// consulting a record. /// @@ -993,7 +933,7 @@ fn remove_interface_in_pass(uri: &str, tap: &str, libvirt: &mut bool) -> Result< /// less than the thing it replaced is not a sweep. Those names are decided /// against a single listing, because the whole point of enumerating a bounded /// space is that deciding one name stays cheap. -fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Result<(usize, bool)> { +fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Result { let identity = InterfaceIdentity { instance_id: instance_id.to_string(), vm_id: vm_id.to_string(), @@ -1009,55 +949,45 @@ fn sweep_vm_interfaces(libvirt_uri: &str, instance_id: &str, vm_id: &str) -> Res // out by trying, once. let mut libvirt = true; let mut removed = 0; - let mut incomplete = false; let mut first_error = None; - let deadline = std::time::Instant::now() + COLLECTION_DEADLINE; for nic_index in 0..=MAX_NIC_INDEX { - if std::time::Instant::now() >= deadline { - warn!(%vm_id, nic_index, "sweep stopped on its deadline"); - incomplete = true; - break; - } let tap = tap_name(&InterfaceIdentity { nic_index, ..identity.clone() }); let present = Path::new("/sys/class/net").join(&tap).exists(); - if !present { - if libvirt - && bindings - .as_ref() - .is_some_and(|bindings| bindings.contains(&tap)) - { - if let Err(error) = delete_binding(libvirt_uri, &tap) { - warn!(%tap, %error, "failed to remove orphaned nwfilter binding"); - libvirt = false; - first_error.get_or_insert(error); - } else { - info!(%tap, %vm_id, "removed orphaned nwfilter binding"); - } + // A pass gets one answer about libvirt, not one per interface. Asking + // again after it has failed is how a hung `libvirtd` turns a bounded + // collection into an unbounded one. + let wanted = present || bindings.as_ref().is_some_and(|held| held.contains(&tap)); + if libvirt && wanted && !is_macvtap(&tap) { + if let Err(error) = delete_binding(libvirt_uri, &tap) { + warn!(%tap, %error, "failed to remove an nwfilter binding"); + libvirt = false; + first_error.get_or_insert(error); + } else if !present { + info!(%tap, %vm_id, "removed orphaned nwfilter binding"); } + } + if !present { continue; } - match remove_interface_in_pass(libvirt_uri, &tap, &mut libvirt) { - // Keep going after a failure. Stopping at the first one would leave - // the rest of a VM's interfaces behind over one that is stuck. + // Keep going after a failure. Stopping at the first one would leave the + // rest of a VM's interfaces behind over one that is stuck. + match remove_interface(libvirt_uri, &tap, BindingCleanup::Skip) { Err(error) => { warn!(%tap, %error, "failed to remove interface"); first_error.get_or_insert(error); } - Ok(cleared) => { + Ok(()) => { info!(%tap, %vm_id, "removed interface"); removed += 1; - if !cleared { - incomplete = true; - } } } } match first_error { Some(error) => Err(error).context("failed to remove every interface for this VM"), - None => Ok((removed, incomplete)), + None => Ok(removed), } } @@ -1096,9 +1026,6 @@ fn list_interfaces(libvirt_uri: &str, instance_id: &str) -> Vec let owner = owner_of(&tap, &alias); seen.insert(tap.clone()); records.push(InterfaceRecord { - bound: bindings - .as_ref() - .is_some_and(|bindings| bindings.contains(&tap)), kind: kind.to_string(), nic_index: owner.as_ref().map(|identity| identity.nic_index), instance_id: owner.as_ref().map(|identity| identity.instance_id.clone()), @@ -1118,7 +1045,6 @@ fn list_interfaces(libvirt_uri: &str, instance_id: &str) -> Vec instance_id: None, vm_id: None, nic_index: None, - bound: true, }); } } @@ -1316,16 +1242,6 @@ fn validate_identity(identity: &InterfaceIdentity) -> Result<()> { Ok(()) } -/// A caller that knows a binding is there needs it gone; one that does not -/// still clears whatever it finds, without failing when libvirt is absent. -fn binding_cleanup(filtered: bool) -> BindingCleanup { - if filtered { - BindingCleanup::Required - } else { - BindingCleanup::BestEffort - } -} - /// Normalizes a requested queue pair count. Zero means the caller did not ask /// for multiqueue, which is the same device shape as one queue pair. fn validate_queues(queues: u32) -> Result { @@ -1730,16 +1646,6 @@ mod tests { assert!(value.get("identity").is_none()); } - /// A binding outlives the interface it was bound to, and TAP names are a - /// deterministic hash of the VM identity, so the same name comes back. - /// Removing an interface therefore clears whatever binding is there, and - /// only insists when the caller is about to create a replacement. - #[test] - fn binding_cleanup_insists_only_when_a_replacement_follows() { - assert_eq!(binding_cleanup(true), BindingCleanup::Required); - assert_eq!(binding_cleanup(false), BindingCleanup::BestEffort); - } - #[test] fn queue_counts_normalize_to_at_least_one_and_stay_bounded() { assert_eq!(validate_queues(0).unwrap(), 1); @@ -2054,14 +1960,7 @@ mod tests { /// means nothing. #[test] fn a_sweep_reports_a_count_and_no_interface() { - let value = serde_json::to_value( - Outcome::Swept { - removed: 3, - incomplete: false, - } - .into_response(), - ) - .unwrap(); + let value = serde_json::to_value(Outcome::Swept { removed: 3 }.into_response()).unwrap(); assert_eq!(value["removed"], 3); assert!(value.get("tap").is_none()); @@ -2278,9 +2177,8 @@ mod tests { assert!(!is_unreachable(&error)); let netd = testing::FakeNetd::spawn(testing::Behavior::handling(&["remove_all"])); - let sweep = remove_all(netd.socket(), "instance", "vm").await.unwrap(); - assert_eq!(sweep.removed, 0); - assert!(!sweep.incomplete); + let removed = remove_all(netd.socket(), "instance", "vm").await.unwrap(); + assert_eq!(removed, 0); } /// Every "an unreachable netd is not a failure" branch in the VMM hangs diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index 7b0bd846c..12bbbf6de 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -189,9 +189,6 @@ type UpdateDialogState = { disk_size: number; image: string; ports: PortFormEntry[]; - /// What the dialog opened with, normalized, so the update can tell whether - /// this request moved the port mappings at all. - originalPorts: VmmTypes.IPortMapping[]; attachAllGpus: boolean; selectedGpus: string[]; updateGpuConfig: boolean; @@ -283,7 +280,6 @@ function createUpdateDialogState(): UpdateDialogState { disk_size: 0, image: '', ports: [], - originalPorts: [], attachAllGpus: false, selectedGpus: [], updateGpuConfig: false, @@ -1148,10 +1144,6 @@ type CreateVmPayloadSource = { disk_size: config.disk_size || 0, image: config.image || '', ports: clonePortMappings(config.ports || []), - // What the dialog opened with. `update_ports` means "this request moved - // the port mappings", and the server refuses some mappings it has to - // keep accepting on a request that only touched memory. - originalPorts: normalizePorts(clonePortMappings(config.ports || [])), attachAllGpus: gpuSelection.attachAll, selectedGpus: gpuSelection.selected, updateGpuConfig: false, @@ -1331,13 +1323,8 @@ type CreateVmPayloadSource = { body.compose_file = composeNeedsUpdate ? await makeUpdateComposeFile() : undefined; body.encrypted_env = encryptedEnvPayload; body.user_config = updated.user_config; - const ports = normalizePorts(updated.ports); - const portsMoved = - JSON.stringify(ports) !== JSON.stringify(updateDialog.value.originalPorts ?? []); - if (portsMoved) { - body.update_ports = true; - body.ports = ports; - } + body.update_ports = true; + body.ports = normalizePorts(updated.ports); body.gpus = updateDialog.value.updateGpuConfig ? configGpu(updated, true) : undefined; if (updated.updateNetworking) { body.update_networking = true; From 5edf2c98e8e582a5b353607d3cc5d9b927625e88 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 09:30:14 -0700 Subject: [PATCH 31/34] refactor(vmm): retry a removal that could not release, instead of collecting after it The leak this PR set out to close is a VM removed while its interfaces could not be released: the release is deliberately non-fatal, so the removal went on to delete the workdir, and with it the only thing on the host that could still name what netd was holding. The answer was a whole-host reconciliation -- `netd list`, a claimable-VM set, a per-VM re-check under the launch lock, and an hourly task -- deciding from the outside which interfaces no VM claims. The VM already knows. `release_vm_interfaces` now says whether the sweep landed, and `finish_remove_vm` deletes the workdir only when it did. If netd refused, or was not there to ask, the directory and its `.removing` marker stay and the next VMM start resumes the removal, which `reload_vms` already knows how to do. `remove_all` is idempotent, so the retry costs one round trip. A VM that never asked netd for an interface is unaffected: the removal reads its recorded networks first, so an absent netd is not a reason to keep the directory of a VM netd never held anything for. Retrying at the next start is also strictly better than the timer it replaces, which waited up to an hour and skipped any VM whose launch lock was held. Gone with it: `reconcile_netd_interfaces`, `claimable_vm_ids`, `claims_vm`, `try_launch_lock`, `netd_reconcile_task`, `netd.reconcile_interval_secs`, and the startup pass -- about 500 lines including tests and docs. An existing `vmm.toml` that still sets the interval keeps loading; nothing denies unknown fields. Also gone is the duplicate-`instance_id` startup refusal and the `instance_id` it added to the discovery record. It caught one configuration and silently passed four -- a peer under a different uid (the discovery directory is per-user while the netd socket is host-wide), a peer registered by an older VMM, two VMMs starting at once, and any later edit -- while its error message taught operators that the collision was checked. `vmm.toml` says plainly that two instances must not share the value, and the derived default cannot collide. Kept: the ownership record. Teardown never needed it -- a sweep derives the names it deletes -- but an operator does, and on a host running several VMM instances it is the only thing that tells one instance's interfaces from another's. `netd list`, `remove-vm` and `remove-interface` are now the whole recovery story for what no VMM will retry: a workdir deleted by hand, or an interface recorded under an instance ID nothing uses any more. --- docs/bridge-networking.md | 62 +++--- dstack/vmm/src/app.rs | 408 +++++------------------------------- dstack/vmm/src/config.rs | 15 -- dstack/vmm/src/discovery.rs | 21 -- dstack/vmm/src/main.rs | 66 +----- dstack/vmm/vmm.toml | 15 +- 6 files changed, 97 insertions(+), 490 deletions(-) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 62abda89d..43216d086 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -255,6 +255,10 @@ authority: a record is believed only when re-deriving the interface name from it reproduces the name it is written on, so a forged, truncated or ambiguous record reads the same as no record at all. +Teardown does not need it — a sweep derives the names it deletes. What needs it +is an operator, and a host running several VMM instances, where it is the only +thing that tells one instance's interfaces from another's. + ```bash # What netd holds on this host sudo dstack-vmm netd list @@ -263,46 +267,34 @@ sudo dstack-vmm netd list sudo dstack-vmm netd remove-vm --instance path-3f9a1c8e7d2b4a60 --vm 0a1b2c3d4e5f6071 ``` -### Collection - -The VMM reconciles what `netd` holds against the VMs it has: once at startup, -after loading them, and then every `netd.reconcile_interval_secs`. That is what -reaches an interface no per-VM teardown can — one whose VM was removed while -the VMM was down, or whose workdir was deleted by hand. - -It asks `netd list` and decides for itself, rather than asking `netd` to decide. -The decision is only safe under the VMM's per-VM launch lock, which `netd` has -no way to take: a collection decided inside `netd` would be decided against a -set of live VMs that was true when the request was *sent*, and `netd` runs it -when it wins the operation lock — possibly much later, by which time a VM -created in between is absent from the set and present on the host. Here each -VM is re-checked while holding the lock its own launch holds, so a launch and a -collection of the same VM cannot both believe they are alone. - -| What the interface is recorded as | What happens | -| --- | --- | -| Another VMM instance's | Never touched. Several VMM instances share one `netd`, and the record is the only thing that can tell that instance's *running* VM from garbage | -| This instance's, for a VM it no longer has | Collected, by the same whole-VM sweep a stop uses | -| Nothing that checks out | Left alone. No VMM can tell whose it is, so no VMM decides about it | - -An interface with no record is not nobody's: before `netd` recorded ownership -every interface looked like this, and on a host with two VMM instances one of -them may be the other's running VM. They are reported at each pass and listed -by `netd list` with `-` for instance and VM; an operator who can tell what one -is removes it by name: +### When a release does not land + +Every stop and every removal asks `netd` to sweep that VM's interfaces, by +deriving each of the 256 names its identity could produce. That needs no +record, and it reaches what a per-NIC teardown cannot: an interface a crash +left behind before anything on disk pointed at it, or one whose NIC the +manifest has since dropped. + +A removal deletes the VM's directory, and that directory — with its `.removing` +marker — is the only thing left that says to try again. So it is deleted only +once the sweep has landed. If `netd` refused, or was not there to ask, the +directory stays and the next VMM start resumes the removal; `remove_all` is +idempotent, so the retry costs one round trip. A VM that never asked `netd` for +an interface is unaffected: there is nothing for `netd` to be holding. + +What no VMM will retry is an interface whose VM directory an operator deleted +by hand, or one recorded under an instance ID no VMM uses any more. `netd list` +shows both, with the instance and VM they are recorded under: ```bash +sudo dstack-vmm netd list +sudo dstack-vmm netd remove-vm --instance --vm sudo dstack-vmm netd remove-interface dtc41d9e0b7a52 ``` -Nothing accumulates: each interface gains a record the next time its VM -launches, so the set only shrinks. - -Changing `cvm.instance_id` — or `run_path`, which it is derived from — is the -one move that strands interfaces on purpose. They stay recorded under the old -namespace, so no VMM collects them and running VMs keep working until they -stop. `netd list` still shows the old instance ID, which is what -`netd remove-vm --instance --vm ` needs. +Changing `cvm.instance_id` — or `run_path`, which it is derived from — strands +interfaces the same way. Running VMs keep working until they stop, and +`netd list` still shows the old instance ID, which is what `remove-vm` needs. ### Mixing networking modes diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 7b8484664..121f6e301 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -367,17 +367,6 @@ impl App { self.launch_lock_handle(id).lock_owned().await } - /// The launch lock, if nothing else holds it. - /// - /// For a caller with something better to do than wait. A held lock means a - /// launch, a stop or a removal of this VM is in flight, and every one of - /// those manages that VM's interfaces itself -- so waiting would be waiting - /// for the very thing that makes the work unnecessary. A removal holds it - /// for as long as the VM takes to exit, which its own comment puts at hours. - pub(crate) fn try_launch_lock(&self, id: &str) -> Option> { - self.launch_lock_handle(id).try_lock_owned().ok() - } - fn launch_lock_handle(&self, id: &str) -> Arc> { let mut locks = self.launch_locks.lock().or_panic("mutex poisoned"); // A VM that is neither starting nor stopping leaves the map holding @@ -850,177 +839,6 @@ impl App { } } - /// Every VM whose interfaces this instance may still be using. - /// - /// Wider than the VMs it managed to load. A VM whose manifest is corrupt - /// or whose image is missing fails to load and is only logged; `reload_vms` - /// then stops any supervisor process it still has, but that runs in the - /// background, and a collection deciding on the loaded set alone races it - /// -- over a file the VMM could not parse. A directory is enough of a - /// claim, and what has been removed for real leaves none behind. - /// `None` when the answer cannot be established, which is not the same as - /// nobody claiming anything: an unreadable VM directory read as empty - /// would offer every interface on the host up for collection. - fn claimable_vm_ids(&self) -> Option> { - let mut ids: HashSet = self.lock().vms.keys().cloned().collect(); - match fs::read_dir(self.vm_dir()) { - Ok(entries) => { - for entry in entries { - // Every error here is answered the same way the `read_dir` - // error below is, and for the same reason: an entry that - // cannot be read is a VM that might exist, and dropping it - // silently is how a collection deletes a running VM's - // networking over an unreadable directory entry. - let entry = match entry { - Ok(entry) => entry, - Err(error) => { - warn!("failed to read a VM directory entry: {error}; not collecting"); - return None; - } - }; - match entry.file_type() { - Ok(file_type) if !file_type.is_dir() => continue, - Ok(_) => {} - Err(error) => { - warn!( - name = ?entry.file_name(), - "failed to stat a VM directory entry: {error}; not collecting" - ); - return None; - } - } - match entry.file_name().into_string() { - Ok(id) => { - ids.insert(id); - } - // A VM ID is ASCII, so a name that is not UTF-8 is not - // a VM directory -- but it is also not something to - // decide a deletion around. - Err(name) => { - warn!(?name, "unreadable VM directory name; not collecting"); - return None; - } - } - } - } - // "Never ran" and "the volume is not mounted yet" produce the same - // error, and only one of them means there are no VMs. A VMM that - // has never run has nothing to collect either way, so declining - // costs nothing and the other reading costs every interface this - // instance owns. - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - debug!("no VM directory yet; not collecting interfaces"); - return None; - } - Err(error) => { - warn!("failed to read the VM directory: {error}; not collecting interfaces"); - return None; - } - } - Some(ids) - } - - /// Deletes every host interface netd holds for a VM this instance no - /// longer has. - /// - /// Per-VM release reaches only what its caller can still name. This reaches - /// what nothing names any more, which is where a leak actually ends up: a - /// VM removed while the VMM was down, a workdir deleted by hand, a teardown - /// that raced a netd outage and was never retried because the VM it - /// belonged to no longer exists to retry it. - /// - /// The decision is made here and not in netd, and that is the whole design. - /// A collection decided in netd is decided against a set of live VMs that - /// was true when the caller sent it: netd runs the request when it wins the - /// operation lock, which may be long after, and a VM created in between is - /// absent from the set and present on the host. netd cannot close that -- - /// the lock that would close it is the VMM's per-VM launch lock, and netd - /// has no way to take it. Here, each VM is decided under exactly that lock - /// and re-checked while holding it, so a launch and a collection of the - /// same VM cannot both believe they are alone. - /// - /// What is claimed is every VM this instance has, running or not. Not the - /// running set: a VMM restarts under VMs that keep running, and collecting - /// by what is running would delete their interfaces out from under them. - pub(crate) async fn reconcile_netd_interfaces(&self) { - let Some(claimed) = self.claimable_vm_ids() else { - return; - }; - let interfaces = match netd::list(&self.config.netd.socket, "").await { - Ok(interfaces) => interfaces, - Err(error) if netd::is_unreachable(&error) => { - debug!("no netd to reconcile interfaces with"); - return; - } - Err(error) => { - warn!( - "failed to list netd-managed interfaces, so nothing is collected this pass: \ - {error:#}" - ); - return; - } - }; - let mut unattributed = 0; - let mut dead = BTreeSet::new(); - for record in &interfaces { - match (&record.instance_id, &record.vm_id) { - // Another instance's. Never ours to collect: on a host where - // two VMMs share one netd, this is the other one's running VM, - // and the ownership record is the only thing that says so. - (Some(instance_id), _) if instance_id != &self.config.cvm.instance_id => {} - (Some(_), Some(vm_id)) if !claimed.contains(vm_id) => { - dead.insert(vm_id.clone()); - } - (Some(_), _) => {} - // Built before netd recorded ownership, or by another netd. - // Nothing here can attribute it, so nothing here can decide - // about it: `dstack-vmm netd remove-interface` is where an - // operator who can decide says so. - _ => unattributed += 1, - } - } - if unattributed > 0 { - info!( - unattributed, - "host interfaces carry no ownership record, so no VMM can tell whose they are \ - and none will collect them; `dstack-vmm netd list` shows them and \ - `netd remove-interface` removes one" - ); - } - for vm_id in dead { - // The lock a launch of this VM holds from before it asks netd for - // an interface until after it has recorded one -- and that a - // removal holds until the VM has exited, which can be hours. Taking - // it without waiting is both safe and necessary: whoever holds it - // is already dealing with this VM's interfaces, and waiting would - // stall the collection of every other VM behind one that is busy. - let Some(_launch) = self.try_launch_lock(&vm_id) else { - debug!(vm_id = %vm_id, "not collecting: this VM is busy"); - continue; - }; - // Re-read under it. Between the listing and this line the VM may - // have been created and started: its directory exists now, and its - // interfaces are the ones a launch just built. - if self.claims_vm(&vm_id) { - continue; - } - info!(vm_id = %vm_id, "collecting host interfaces no VM of this instance claims"); - self.release_vm_interfaces(&vm_id).await; - } - } - - /// Whether this instance has a VM by this ID at all, loaded or merely - /// present on disk. See [`App::claimable_vm_ids`]. - fn claims_vm(&self, vm_id: &str) -> bool { - if self.lock().vms.contains_key(vm_id) { - return true; - } - // `validate_vm_id` keeps an ID from naming anything outside the VM - // directory, so this cannot be pointed at another path. - self.work_dir(vm_id) - .is_ok_and(|work_dir| work_dir.path().is_dir()) - } - /// Releases every host interface netd holds for this VM. /// /// Unconditional and non-fatal, which is one decision made twice. The @@ -1031,11 +849,11 @@ impl App { /// stop when the daemon holding its interfaces cannot be reached, or a /// netd outage becomes a fleet that cannot be stopped. /// - /// Nothing is lost by not failing: the next launch releases before it - /// prepares, and startup reconciliation collects what no launch will ever - /// reach. `recorded` is not the source of truth -- it is what a netd too - /// old to sweep by identity has to be told instead. - pub(crate) async fn release_vm_interfaces(&self, vm_id: &str) { + /// Returns whether netd is known to hold nothing for this VM any more. + /// A removal reads that to decide whether it may delete the workdir: the + /// directory is what says to try again, and deleting it over a failed + /// release is what strands an interface with nothing left to reach it. + pub(crate) async fn release_vm_interfaces(&self, vm_id: &str) -> bool { // Ask for the release, rather than asking whether it can be asked for. // A probe first would put a second round trip in front of every stop // and -- worse -- would make a *busy* netd look like an absent one and @@ -1053,17 +871,18 @@ impl App { if removed > 0 { info!(vm_id, removed, "released netd-managed interfaces"); } + true } Err(error) if netd::is_unreachable(&error) => { - debug!(vm_id, %error, "no netd to release interfaces from") + debug!(vm_id, %error, "no netd to release interfaces from"); + false } Err(error) => { - // Whatever this VM holds stays until it launches again, or - // until reconciliation collects it once nothing claims it. warn!( vm_id, "failed to release netd-managed interfaces: {error:#}" ); + false } } } @@ -1191,22 +1010,37 @@ impl App { } } - self.release_vm_interfaces(id).await; + let vm_path = self.work_dir(id)?; + // Read before the release, because the release is what makes it stale. + // A VM that never asked netd for an interface -- user mode, a custom + // netdev, or one that never launched -- has nothing for netd to be + // holding, so an absent netd is not a reason to keep its directory. + let held_interfaces = vm_path.runtime_networks().iter().any(needs_netd_interface); + let released = self.release_vm_interfaces(id).await; // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. - let vm_path = self.work_dir(id)?; - if delete_workdir || vm_path.is_removing() { + if !(delete_workdir || vm_path.is_removing()) { if vm_path.path().exists() { - if let Err(err) = fs::remove_dir_all(&vm_path) { - error!("failed to remove VM directory for {id}: {err:?}"); - } + info!( + "VM {id} workdir preserved (orphan cleanup): {}", + vm_path.path().display() + ); } - } else if vm_path.path().exists() { - info!( - "VM {id} workdir preserved (orphan cleanup): {}", - vm_path.path().display() + } else if held_interfaces && !released { + // The `.removing` marker and the directory are what a later boot + // reads to retry this, and `remove_all` is idempotent, so keeping + // them costs one retry and losing them strands every interface + // netd still holds: nothing else on the host can name them. + warn!( + "VM {id} keeps its directory because netd did not release its interfaces; \ + the removal resumes at the next VMM start" ); + return Ok(()); + } else if vm_path.path().exists() { + if let Err(err) = fs::remove_dir_all(&vm_path) { + error!("failed to remove VM directory for {id}: {err:?}"); + } } // Free CID and remove from memory (last step) @@ -2334,43 +2168,6 @@ mod tests { ) } - /// A VM whose manifest is corrupt or whose image is missing fails to load - /// and is only logged -- but its QEMU may well still be running. Collecting - /// by what loaded would delete a running VM's networking over a file the - /// VMM could not parse. - #[test] - fn a_vm_that_failed_to_load_still_claims_its_interfaces() { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir(dir.path().join("vm-that-did-not-load")).unwrap(); - std::fs::write(dir.path().join("not-a-vm"), "").unwrap(); - let app = App::new( - test_config(Path::new("/nonexistent/netd.sock"), dir.path()), - SupervisorClient::new("http://127.0.0.1:0"), - ); - assert_eq!( - app.claimable_vm_ids(), - Some(HashSet::from(["vm-that-did-not-load".to_string()])) - ); - - // Unreadable is not "nobody claims anything", which would offer every - // interface on the host up for collection. - let app = App::new( - test_config( - Path::new("/nonexistent/netd.sock"), - Path::new("/proc/self/environ"), - ), - SupervisorClient::new("http://127.0.0.1:0"), - ); - assert_eq!(app.claimable_vm_ids(), None); - - // "Never ran" and "the volume is not mounted yet" produce the same - // error and only one of them means there are no VMs. A VMM that has - // never run has nothing to collect either way, so declining costs - // nothing and the other reading costs every interface it owns. - let app = app_talking_to(Path::new("/nonexistent/netd.sock")); - assert_eq!(app.claimable_vm_ids(), None); - } - /// A netd outage must not become a fleet that cannot be stopped. #[tokio::test] async fn a_stop_survives_a_netd_that_is_not_there() { @@ -2380,6 +2177,29 @@ mod tests { app.release_vm_interfaces("vm-1").await; } + /// A removal deletes the workdir, and the workdir is the only thing left + /// that says to retry. So the release has to say whether it landed: an + /// answer means netd holds nothing, and anything else -- a refusal, or a + /// netd that is not there to ask -- means it may still. + #[tokio::test] + async fn a_release_says_whether_netd_still_holds_anything() { + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); + let app = app_talking_to(netd.socket()); + assert!(app.release_vm_interfaces("vm-1").await); + + // Refused: netd is up and still holding whatever it had. + let refusing = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); + let app = app_talking_to(refusing.socket()); + assert!(!app.release_vm_interfaces("vm-1").await); + + // Not there to ask. A VM that never asked netd for an interface is + // unaffected -- the removal checks that separately -- but one that did + // must keep its directory so a later start can try again. + let app = app_talking_to(Path::new("/nonexistent/dstack-netd.sock")); + assert!(!app.release_vm_interfaces("vm-1").await); + } + /// A netd too old for the sweep refuses it, and a refusal is not a reason /// to fail the stop. What it holds stays until this VM launches again or /// until reconciliation collects it, which is what an operator upgrading @@ -2416,120 +2236,6 @@ mod tests { assert!(sweep.get("nic_index").is_none()); } - fn held(tap: &str, instance: Option<&str>, vm: Option<&str>) -> serde_json::Value { - serde_json::json!({ - "tap": tap, - "kind": "tap", - "instance_id": instance, - "vm_id": vm, - "nic_index": 0, - }) - } - - /// The decision a collection is made of. It lives here, and not in netd, - /// because it is only safe under a lock netd cannot take -- see - /// [`App::reconcile_netd_interfaces`]. - #[tokio::test] - async fn a_collection_takes_only_what_this_instance_no_longer_claims() { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir(dir.path().join("live-vm")).unwrap(); - let netd = netd::testing::FakeNetd::spawn_holding( - netd::testing::Behavior::handling(&["list", "remove_all"]), - vec![ - held("dt000000000001", Some("test-instance"), Some("live-vm")), - held("dt000000000002", Some("test-instance"), Some("dead-vm")), - // Another VMM instance on the same host. The ownership record - // is the only thing that can tell this from ours, which is why - // there is one: without it a collection would delete another - // instance's running VM's networking. - held("dt000000000003", Some("someone-else"), Some("dead-vm")), - // Built before ownership was recorded, or by another netd. - // Nothing here can attribute it, so nothing here decides. - held("dt000000000004", None, None), - ], - ); - let app = App::new( - test_config(netd.socket(), dir.path()), - SupervisorClient::new("http://127.0.0.1:0"), - ); - app.reconcile_netd_interfaces().await; - - let swept: Vec = netd - .seen() - .into_iter() - .filter(|request| request["operation"] == "remove_all") - .collect(); - assert_eq!(swept.len(), 1, "one VM collected, and only one"); - assert_eq!(swept[0]["vm_id"], "dead-vm"); - assert_eq!(swept[0]["instance_id"], "test-instance"); - } - - /// A netd that cannot say what it holds must not read as one holding - /// nothing. - #[tokio::test] - async fn a_netd_that_cannot_enumerate_collects_nothing() { - let dir = tempfile::tempdir().unwrap(); - let netd = - netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); - let app = App::new( - test_config(netd.socket(), dir.path()), - SupervisorClient::new("http://127.0.0.1:0"), - ); - app.reconcile_netd_interfaces().await; - assert!( - !netd - .operations() - .iter() - .any(|operation| operation == "remove_all"), - "nothing is collected on the strength of an answer netd could not give" - ); - } - - /// A VM directory that cannot be read is not an absent VM. Reading it as - /// one offers every interface this instance owns up for collection. - #[tokio::test] - async fn an_unreadable_vm_directory_collects_nothing() { - let netd = netd::testing::FakeNetd::spawn_holding( - netd::testing::Behavior::handling(&["list", "remove_all"]), - vec![held( - "dt000000000001", - Some("test-instance"), - Some("running-vm"), - )], - ); - for run_path in [ - Path::new("/proc/self/environ"), - Path::new("/nonexistent/vms"), - ] { - let app = App::new( - test_config(netd.socket(), run_path), - SupervisorClient::new("http://127.0.0.1:0"), - ); - app.reconcile_netd_interfaces().await; - } - assert!( - netd.operations().is_empty(), - "an answer that could not be established is not an answer" - ); - } - - /// A removal holds the launch lock until the VM has exited, which its own - /// comment puts at hours. Nothing that has something better to do than - /// wait may queue behind it. - #[tokio::test] - async fn work_that_can_wait_does_not_queue_behind_a_removal() { - let app = test_app(); - let removal = app.launch_lock("vm-1").await; - - // The collection skips a busy VM rather than stalling every other VM - // behind it: whoever holds the lock is already dealing with this one. - assert!(app.try_launch_lock("vm-1").is_none()); - assert!(app.try_launch_lock("vm-2").is_some()); - - drop(removal); - assert!(app.try_launch_lock("vm-1").is_some()); - } - /// The orphan cleanup runs for an ID that never loaded, so a guard living /// in the VM entry is not there when it holds the launch lock across the /// whole teardown. Without the mark, `StartVm` on that ID waits the diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 834dc0673..9ec20a115 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -644,20 +644,6 @@ pub struct NetdConfig { /// left inferring policy from a file that does not state it. #[serde(default)] pub network_filter: Option, - /// How often the VMM asks netd to collect interfaces no VM of its claims. - /// - /// The pass at startup is not optional and does not read this: it is the - /// one moment the VMM knows every VM it has, and everything a crash left - /// behind is still there. This is the backstop for what accumulates while - /// it runs -- a removal that raced a netd outage, an interface a stop could - /// not reach. Zero turns it off. - #[serde(default = "default_reconcile_interval")] - pub reconcile_interval_secs: u64, -} - -/// See [`NetdConfig::reconcile_interval_secs`]. -fn default_reconcile_interval() -> u64 { - 3600 } impl Default for NetdConfig { @@ -667,7 +653,6 @@ impl Default for NetdConfig { socket_mode: 0o660, libvirt_uri: default_libvirt_uri(), network_filter: None, - reconcile_interval_secs: default_reconcile_interval(), } } } diff --git a/dstack/vmm/src/discovery.rs b/dstack/vmm/src/discovery.rs index f17610c18..181e7da52 100644 --- a/dstack/vmm/src/discovery.rs +++ b/dstack/vmm/src/discovery.rs @@ -42,11 +42,6 @@ pub struct VmmInstanceInfo { pub run_path: String, /// Node name from configuration. pub node_name: String, - /// The namespace this VMM's host interfaces are recorded under. Two live - /// instances must not share one: it is the name space their TAP names are - /// derived in and the only thing a collection can tell them apart by. - #[serde(default)] - pub instance_id: String, /// VMM version string. pub version: String, /// Unix timestamp (seconds) when the instance started. @@ -69,7 +64,6 @@ impl DiscoveryRegistration { run_path: &Path, node_name: &str, version: &str, - instance_id: &str, ) -> Result { let dir = discovery_dir(); fs_err::create_dir_all(&dir).context("failed to create discovery directory")?; @@ -88,7 +82,6 @@ impl DiscoveryRegistration { image_path: image_path.to_string_lossy().to_string(), run_path: run_path.to_string_lossy().to_string(), node_name: node_name.to_string(), - instance_id: instance_id.to_string(), version: version.to_string(), started_at: std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -117,20 +110,6 @@ impl Drop for DiscoveryRegistration { } } -/// Every VMM instance currently registered as alive on this host. -pub fn live_instances() -> Vec { - let Ok(entries) = fs::read_dir(discovery_dir()) else { - return Vec::new(); - }; - entries - .flatten() - .filter(|entry| entry.path().extension().and_then(|e| e.to_str()) == Some("json")) - .filter_map(|entry| fs::read_to_string(entry.path()).ok()) - .filter_map(|content| serde_json::from_str::(&content).ok()) - .filter(|info| Path::new(&format!("/proc/{}", info.pid)).exists()) - .collect() -} - /// Clean up stale discovery files from dead processes. pub fn cleanup_stale_registrations() { let dir = discovery_dir(); diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 1985a2a39..d35864dc4 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -96,11 +96,10 @@ enum NetdCommand { }, /// Delete one interface by name. /// - /// For what nothing else can reach: an interface built before netd - /// recorded ownership, or by another netd, whose VM is gone. `netd list` - /// shows these with no instance and no VM -- nothing can attribute them, - /// so no VMM will ever collect them, and an operator who can tell what - /// they are says so here. + /// For what nothing else can name: an interface built before netd recorded + /// ownership, or by another netd, whose VM is gone. `netd list` shows these + /// with no instance and no VM, so nothing can derive the sweep that would + /// take them; an operator who can tell what they are says so here. RemoveInterface { /// The interface name, as `netd list` prints it. name: String, @@ -108,8 +107,9 @@ enum NetdCommand { /// Delete every interface netd holds for one VM. /// /// For a VM whose VMM will never ask again -- one whose directory was - /// deleted by hand, or whose instance is gone. A running VMM collects - /// these itself; this is for when there is no longer one to do it. + /// deleted by hand, or whose instance is gone. A VMM sweeps its own VMs on + /// every stop and every removal, and keeps a removal pending until the + /// sweep lands; this is for when no VMM will ever run that sweep. RemoveVm { /// The `cvm.instance_id` of the VMM that created them. `netd list` /// shows it. @@ -269,7 +269,8 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< // recorded ownership, or by another netd, carries no record and // gets one the next time its VM launches. println!( - "{unattributed} carry no ownership record, so a collection will not touch them" + "{unattributed} carry no ownership record; `netd remove-interface` takes \ + one by name" ); } Ok(()) @@ -291,27 +292,6 @@ async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result< } } -/// Collects host interfaces no VM claims, on an interval. -/// -/// The startup pass covers what a crash left behind. This covers what -/// accumulates while the VMM runs: a removal that raced a netd outage, a -/// teardown whose VM no longer exists to retry it. -async fn netd_reconcile_task(app: App) { - let interval_secs = app.config.netd.reconcile_interval_secs; - if interval_secs == 0 { - info!("periodic netd reconciliation is disabled"); - return; - } - let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); - // The startup pass already ran, and this fires immediately on its first - // tick. - interval.tick().await; - loop { - interval.tick().await; - app.reconcile_netd_interfaces().await; - } -} - #[rocket::main] async fn main() -> Result<()> { { @@ -370,27 +350,6 @@ async fn main() -> Result<()> { // Preserve the existing startup validation. The broader static checks are // opt-in through `check-config` until they have seen wider deployment use. netd::validate_instance_id(&config.cvm.instance_id)?; - // Two live VMMs sharing one instance ID share the name space their host - // interfaces are derived in: each would build TAPs at names the other can - // also produce, and each collection would delete the other's running VMs - // because the ownership record -- the only thing that can tell two - // instances apart -- would name the collector. Derived from `run_path` - // this cannot happen; it takes a copied `vmm.toml` that states one. - for peer in discovery::live_instances() { - if peer.instance_id == config.cvm.instance_id - && peer.run_path != config.run_path.to_string_lossy() - { - anyhow::bail!( - "cvm.instance_id '{}' is already in use by the VMM running at {} (pid {}). It is \ - the name space this VMM's host interfaces are derived in, so sharing one would \ - have each instance delete the other's running VMs' networking. Leave it empty to \ - derive it from run_path", - config.cvm.instance_id, - peer.run_path, - peer.pid - ); - } - } config .host_api .validate() @@ -458,7 +417,6 @@ async fn main() -> Result<()> { &config.run_path, &config.node_name, &app_version(), - &config.cvm.instance_id, ) { Ok(registration) => Some(registration), Err(err) => { @@ -499,14 +457,8 @@ async fn main() -> Result<()> { }; let state = app::App::new(config, supervisor); state.reload_vms().await.context("Failed to reload VMs")?; - // After the VMs are loaded, because the set of VMs this instance has is - // what the collection is decided against, and before the API is served, - // because a VM created between taking that set and acting on it would be - // in netd's listing and not in the set. - state.reconcile_netd_interfaces().await; tokio::spawn(auto_restart_task(state.clone())); tokio::spawn(log_rotation_task(state.clone())); - tokio::spawn(netd_reconcile_task(state.clone())); tokio::select! { result = run_external_api(state.clone(), figment.clone(), api_auth) => { diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 6be8400aa..9ef0ecec4 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -47,11 +47,10 @@ qmp_socket = false # The user to run the VM as. If empty, the VM will be run as the current user. # Unique namespace when multiple dstack-vmm instances share one host. When # empty, a stable value is derived from run_path. It is what netd records on -# every host interface this VMM asks for, and what a collection uses to tell -# this instance's interfaces from another's -- so two VMMs sharing one value on -# one host would each collect the other's running VMs. A VMM that finds another -# live instance using the value it was given refuses to start rather than let -# that happen. May not contain ":". +# every host interface this VMM asks for, and the name space those interfaces' +# names are derived in. Two VMMs on one host must not share one: each would +# build interfaces at names the other can also produce. Left empty it is +# derived from run_path, which cannot collide. May not contain ":". instance_id = "" # Network choices that deployment RPC callers may make. Macvtap is excluded # by default because libvirt nwfilter applies only to bridge interfaces. @@ -155,12 +154,6 @@ parameters = {} # permissions authorize clients. [netd] socket = "/run/dstack/netd.sock" -# How often the VMM asks netd to collect interfaces no VM of its claims. The -# pass at startup is not optional and does not read this: it is the one moment -# the VMM knows every VM it has and everything a crash left behind is still -# there. This is the backstop for what accumulates while it runs. 0 disables it. -reconcile_interval_secs = 3600 - # Applied when netd creates the socket itself. A systemd socket unit controls # its own SocketMode instead. socket_mode = 0o660 From b32f1a63bf4f35f91b9aca6f353d4c45466cef74 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 09:36:45 -0700 Subject: [PATCH 32/34] docs(vmm): correct three comments the collection removal left behind --- dstack/vmm/src/app.rs | 10 +++++----- dstack/vmm/src/netd.rs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 121f6e301..f6deaa8ec 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -605,9 +605,9 @@ impl App { // Not fallible: a VM that has been asked to stop is stopped whether or // not netd could be reached. What is left behind is reclaimed by this // VM's next launch, which releases before it prepares, or by its - // removal. Not by reconciliation -- a stopped VM is still one this - // instance claims, so a VM that is never started or removed again keeps - // its interfaces. See [`App::release_vm_interfaces`]. + // removal, which will not finish until the release lands. A VM that is + // never started or removed again keeps its interfaces, and its + // directory is still there to say whose they are. self.release_vm_interfaces(id).await; Ok(()) } @@ -2202,8 +2202,8 @@ mod tests { /// A netd too old for the sweep refuses it, and a refusal is not a reason /// to fail the stop. What it holds stays until this VM launches again or - /// until reconciliation collects it, which is what an operator upgrading - /// netd gets for free. + /// until an operator names it. The stop still succeeds: a netd outage must + /// not become a fleet that cannot be stopped. #[tokio::test] async fn a_netd_that_refuses_the_sweep_does_not_fail_the_stop() { let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 0f8e07d6c..40aab7476 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -156,8 +156,8 @@ pub enum Request { instance_id: String, vm_id: String, }, - /// Everything netd holds, so that an operator -- and a reconciliation -- - /// can see the host's interfaces without being told what to look for. + /// Everything netd holds, so that an operator can see the host's + /// interfaces without being told what to look for. /// /// Deriving a name answers "where is this VM's interface". It cannot /// answer "whose is this interface", which is the question a leak is made From 2496aa5ab3158d955a8b1d4f04a07ca76b00185f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 19:02:02 -0700 Subject: [PATCH 33/34] docs(netd): correct two comments the ingress removal left behind --- dstack/vmm/src/netd.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 40aab7476..aca287a9c 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -202,10 +202,6 @@ struct Response { /// between "one queue was requested" and "this netd ignored the request". #[serde(default, skip_serializing_if = "Option::is_none")] queues: Option, - /// Forwarding rules netd established. Absent from a netd that does not - /// forward host ports, which is how the caller tells "nothing was asked - /// for" apart from "this request was ignored" -- the same reading `queues` - /// gets above. /// How many interfaces a whole-VM sweep deleted. #[serde(default, skip_serializing_if = "Option::is_none")] removed: Option, @@ -2164,9 +2160,9 @@ mod tests { remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); } /// A netd that answers a sweep with no count did not sweep. Reading the - /// absent field as zero is the same conflation `queues` and `ingress` are - /// shaped to avoid, and here it would report a netd that cannot collect a - /// VM's interfaces as a VM that had none. + /// absent field as zero is the same conflation `queues` is shaped to + /// avoid, and here it would report a netd that cannot collect a VM's + /// interfaces as a VM that had none. #[tokio::test] async fn a_sweep_without_a_count_is_not_read_as_an_empty_one() { // Claims the operation, answers without the field. From db6e2446d6720c3dbdd20eb579e9f0080476f643 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 4 Sep 2026 19:03:02 -0700 Subject: [PATCH 34/34] docs(vmm): correct two more comments the ingress removal left behind --- dstack/vmm/rpc/proto/vmm_rpc.proto | 3 ++- dstack/vmm/src/app/network.rs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 239ac2d79..62bf381be 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -184,7 +184,8 @@ message PortMapping { string host_address = 4; // Which NIC this mapping's traffic enters through, as an index into // `networks`. Unset picks the first user-mode NIC, which is where QEMU's - // hostfwd entries have always gone, and failing that the first bridge NIC. + // hostfwd entries have always gone and the only place a host port is + // published from. // // A VM with one NIC never needs it. With several there is a choice, and it // used to be made silently: a bridge NIC for external traffic beside a diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index e9145d3dd..b946e3ca1 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -225,9 +225,9 @@ pub(crate) fn mode_carries_ingress(mode: NetworkingMode) -> bool { /// Which NIC a port mapping's traffic enters through. /// -/// One mapping resolves to at most one NIC, and that NIC's backend decides the -/// mechanism: `hostfwd=` for user mode, netd for a bridge. That is what keeps -/// QEMU and netd from both claiming one host port. +/// One mapping resolves to at most one NIC, and only a user-mode NIC has a +/// mechanism to carry it, so a pin to any other kind resolves to nothing +/// rather than to a NIC with no path into the guest. /// /// `None` is a mapping with nowhere to go: a VM with no user-mode NIC, or one /// whose NICs changed under a mapping that named one. The launch warns about