diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 0d16fd476..f627851dc 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -4,7 +4,7 @@ By default, dstack-vmm uses **user** networking (QEMU's built-in SLIRP stack, no ## When to use bridge networking -- High connection concurrency (passt becomes CPU-bound at ~25K+ concurrent connections) +- High connection concurrency (user-mode networking becomes CPU-bound at ~25K+ concurrent connections) - Workloads that need full L2 network access - Environments where VMs need to be directly reachable on the LAN @@ -21,11 +21,11 @@ bridge = "virbr0" ### Per-VM override Individual VMs can override the global networking mode via: -- **CLI**: `vmm-cli.py deploy --net bridge` or `--net passt` +- **CLI**: `vmm-cli.py deploy --net bridge`, `--net user`, or `--net macvtap` - **Web UI**: Networking dropdown in the deploy dialog - **API**: `networking: { mode: "bridge" }` in `VmConfiguration` -Only the mode is per-VM; the bridge interface name always comes from the global config. +The bridge interface name comes from the global config unless the node lists it in `cvm.allowed_bridges`. VMs may also override the vhost and queue settings — see [network-data-plane.md](network-data-plane.md). ## Host setup @@ -143,28 +143,37 @@ mode = "bridge" bridge = "dstack-br0" ``` -### QEMU bridge helper setup (required for both options) +### netd is required -The bridge helper allows QEMU to create and attach TAP devices without VMM needing root privileges. +Bridge networking needs `netd`, the privileged helper that owns every host +interface a bridge or macvtap NIC uses. It is the same binary: ```bash -# Allow QEMU to use the bridge -sudo mkdir -p /etc/qemu -echo "allow virbr0" | sudo tee /etc/qemu/bridge.conf -# Or for manual bridge: echo "allow dstack-br0" | sudo tee /etc/qemu/bridge.conf - -# Set setuid on bridge helper -sudo chmod u+s /usr/lib/qemu/qemu-bridge-helper +sudo dstack-vmm --config vmm.toml netd ``` +Nothing else on the node needs `CAP_NET_ADMIN`: the VMM itself still runs +unprivileged, and `netd` holds the privilege behind a Unix socket whose +filesystem permissions authorize callers. + +This used to be conditional — `netd` built the TAP when libvirt filtering was on +or when the NIC wanted more than one queue pair, and otherwise QEMU's setuid +`qemu-bridge-helper` did. Two owners meant two answers to the same questions: +which netdev QEMU gets, whether vhost is really on, and what a bridge NIC's TAP +is built with. So a bridge NIC's host interface has one owner now, on every +node. + +`qemu-bridge-helper` is no longer used, and `/etc/qemu/bridge.conf` no longer +needs an `allow` line for the bridge. + ## How it works -- VMM passes `-netdev bridge,id=net0,br=` to QEMU -- QEMU's bridge helper (setuid) creates a TAP device and attaches it to the bridge +- `netd` creates a persistent TAP, attaches it to the bridge, binds the nwfilter if the node filters, and the VMM passes `-netdev tap,id=net0,ifname=,...` - Guest MAC address is derived from SHA256 of the VM ID, with an optional configurable prefix (stable across restarts for DHCP IP consistency) - The host DHCP server (dnsmasq) assigns an IP to the VM -- When QEMU exits, the TAP device is automatically destroyed -- VMM does not need root or `CAP_NET_ADMIN` +- The TAP outlives QEMU and is deleted when the VMM tears the VM's networking down, so a VM that crashes does not leave its filter rules attached to a name the next VM could take +- Every interface `netd` creates records which VM of which VMM instance it belongs to, in the kernel's interface alias — see [Who owns an interface](#who-owns-an-interface) +- The VMM process needs neither root nor `CAP_NET_ADMIN`; `netd` holds that privilege in a separate service ### MAC address prefix @@ -192,15 +201,124 @@ The remaining bytes are derived from the VM ID hash. The prefix applies to all n - Docker's nftables chains (`DOCKER-FORWARD`) run before libvirt's but do not block virbr0 traffic - Use `setup-bridge.sh check --bridge ` to diagnose missing rules +### Which NIC a port mapping uses + +A port mapping says which NIC its traffic enters through: + +```bash +vmm-cli.py deploy ... --port udp:0.0.0.0:7483:51820@0 --port tcp:127.0.0.1:7484:8001@0 +``` + +Leave `@` off and the VMM picks the first user-mode NIC — where QEMU's +`hostfwd=` entries have always gone. If the VM has no user-mode NIC, the mapping +has no publishing backend and the launch log names it as stranded. A single-NIC +user-mode VM never needs the suffix. + +With several NICs the choice used to be made silently, and not always the way an +operator would have. A bridge NIC for external traffic beside a user-mode NIC for +management — the topology multi-NIC was added for — put every published port on +the *management* NIC: the traffic reached the guest, but over slirp, bypassing +whatever the bridge NIC's nwfilter was there to enforce and hiding the client's +address behind the slirp gateway. A second user-mode NIC could never publish +anything at all, because only the first was ever selected. + +A mapping resolves to at most one NIC. The only backend that can carry it is +QEMU user networking through `hostfwd=`; `netd` builds bridge interfaces but +does not publish host ports. + +### Which ports a bridge NIC can publish + +QEMU publishes a port with `hostfwd=` on a user-mode NIC, and that is the only +mechanism this host has. **The `netd` in this repository builds interfaces; it +does not forward host ports**, so a bridge NIC cannot carry a port mapping. + +`--port …@` therefore only ever names a user-mode NIC. Pinning to a bridge, +macvtap or custom NIC is refused at deployment, where the caller is there to be +told. An unpinned mapping goes to the first user-mode NIC; a VM that has none is +not refused — it may have been deployed before this — but every mapping it +strands is named in the launch log. + +## Who owns an interface + +`netd` names an interface `dt<12 hex>`, a digest of (VMM instance, VM, NIC +index). That answers "where is this VM's interface" but not "whose is this +interface" — and the second question is the one a leaked interface poses. So +`netd` also records the identity on the interface itself: + +```console +$ ip -d link show dtc41d9e0b7a52 | grep alias + alias dstack1:0:path-3f9a1c8e7d2b4a60:0a1b2c3d4e5f6071 +``` + +The kernel holds that for exactly the interface's lifetime, so unlike a file on +disk it cannot be written late, lost, or left behind. It is a hint, never an +authority: a record is believed only when re-deriving the interface name from +it reproduces the name it is written on, so a forged, truncated or ambiguous +record reads the same as no record at all. + +Teardown does not need it — a sweep derives the names it deletes. What needs it +is an operator, and a host running several VMM instances, where it is the only +thing that tells one instance's interfaces from another's. + +```bash +# What netd holds on this host +sudo dstack-vmm netd list + +# Everything one VM holds, for a VM whose VMM will never ask again +sudo dstack-vmm netd remove-vm --instance path-3f9a1c8e7d2b4a60 --vm 0a1b2c3d4e5f6071 +``` + +### When a release does not land + +Every stop and every removal asks `netd` to sweep that VM's interfaces, by +deriving each of the 256 names its identity could produce. That needs no +record, and it reaches what a per-NIC teardown cannot: an interface a crash +left behind before anything on disk pointed at it, or one whose NIC the +manifest has since dropped. + +A removal deletes the VM's directory, and that directory — with its `.removing` +marker — is the only thing left that says to try again. So it is deleted only +once the sweep has landed. If `netd` refused, or was not there to ask, the +directory stays and the next VMM start resumes the removal; `remove_all` is +idempotent, so the retry costs one round trip. A VM that never asked `netd` for +an interface is unaffected: there is nothing for `netd` to be holding. + +The VMM persists `.netd-pending` before asking netd to prepare an interface and +clears it only after a successful whole-VM sweep. This cleanup marker survives +failed launches and network configuration changes, even if the runtime snapshot +is absent or replaced by a user-mode topology. Older snapshots are promoted to +the marker before cleanup or replacement. A failed cleanup during an update +also leaves the old snapshot intact. + +On an unfiltered node, an unavailable `libvirtd` does not make an otherwise +successful TAP sweep fail. Filtered nodes still require confirmation that their +nwfilter bindings have been released; deleting the TAP alone is not sufficient. + +What no VMM will retry is an interface whose VM directory an operator deleted +by hand, or one recorded under an instance ID no VMM uses any more. `netd list` +shows both, with the instance and VM they are recorded under: + +```bash +sudo dstack-vmm netd list +sudo dstack-vmm netd remove-vm --instance --vm +sudo dstack-vmm netd remove-interface dtc41d9e0b7a52 +``` + +Changing `cvm.instance_id` — or `run_path`, which it is derived from — strands +interfaces the same way. Running VMs keep working until they stop, and +`netd list` still shows the old instance ID, which is what `remove-vm` needs. + ### Mixing networking modes -Bridge and passt VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: +Bridge and user-mode VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: ```bash -# Global default is bridge, but deploy this VM with passt -vmm-cli.py deploy --name my-vm --image dstack-0.5.6 --compose app.yaml --net passt +# Global default is bridge, but deploy this VM with user networking +vmm-cli.py deploy --name my-vm --image dstack-0.5.6 --compose app.yaml --net user ``` -### vhost-net and TDX +### vhost-net and multiqueue + +Bridge NICs can run on the host kernel's vhost-net data plane and expose several virtio-net queue pairs. Both are off by default and enabled per node or per VM — see [network-data-plane.md](network-data-plane.md) for the knobs, the enablement checklist, the mode support matrix, and how to pick a queue count. -vhost-net (kernel data plane offload for virtio-net) is **not enabled** for bridge mode. TDX encrypts guest memory, which prevents the host kernel from performing DMA-based packet offload. The default QEMU userspace virtio backend is used instead. +vhost-net works in a TDX guest: the virtio rings and buffers live in shared, unencrypted memory so that a host-side backend can reach them, which is the same mechanism `vhost-vsock-pci` has always relied on. diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index 942146f0f..3a1313aef 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -14,10 +14,17 @@ host mechanism. The measurable acceptance criteria are: -- `network_filter = "none"` preserves the existing QEMU `-netdev bridge` - behavior and does not require `netd` or libvirt. +- `network_filter = "none"` installs no nwfilter binding. It does not remove + the `netd` dependency: `netd` creates the TAP for every bridge NIC either + way, and the VMM uses `-netdev tap` either way. What changes is only whether + that TAP carries a binding. - `network_filter = "libvirt"` creates the TAP and filter binding before QEMU is submitted to Supervisor, and uses QEMU `-netdev tap`. +- An nwfilter binding outlives the TAP it was bound to, so a teardown clears + the binding at every name that VM could have used, whether or not the + interface is still there. `dstack-vmm netd list` shows a binding whose + interface is already gone as a `binding` row; remove one with + `dstack-vmm netd remove-interface `. - A failed TAP or filter setup prevents QEMU from starting and rolls back all interfaces prepared for that VM. - Normal stop and removal delete the filter binding and TAP. @@ -53,8 +60,11 @@ allowed_macvtap_parents = [] Macvtap is excluded from `allowed_network_modes` by default. Empty bridge and macvtap-parent allowlists prevent RPC callers from overriding the respective -node defaults. If macvtap is explicitly enabled, callers may select only a -parent in `allowed_macvtap_parents`; the macvtap forwarding mode always comes +node defaults. If macvtap is explicitly enabled, callers may select a +parent listed in `allowed_macvtap_parents`, the node's own configured parent, or +one this VM already holds — restating a value the node would have supplied +anyway grants nothing new. The same applies to `bridge_name` and +`allowed_bridges`. The macvtap forwarding mode always comes from `[cvm.networking].macvtap_mode` and cannot be selected through deployment RPCs. These allowlists authorize attachment targets; an nwfilter is not a substitute for that authorization. @@ -82,7 +92,8 @@ For libvirt mode, startup is: 2. Create the TAP for the configured QEMU UID and attach it to the bridge. 3. Create a libvirt nwfilter binding for the TAP. 4. Bring the TAP up and return success. -5. Start QEMU directly with `-netdev tap,script=no,downscript=no`. +5. Start QEMU directly with `-netdev tap,script=no,downscript=no`, carrying + `vhost=on|off` and, above one queue pair, `queues=N`. Teardown stops QEMU first, removes the binding, and deletes the TAP. Operations are serialized by `netd`. The design intentionally does not add ownership @@ -98,11 +109,31 @@ arguments. It never accepts a command, executable path, TAP name, or raw XML from a client. Filter XML is generated internally with XML escaping and is validated by libvirt. +Teardown by identity only reaches the NIC indices its caller still has a record +of, and that record is written *after* the interface exists — a VMM killed in +between leaves a TAP nothing on disk points at, and a manifest that lost a NIC +leaves the same thing behind. `remove_all` names a VM instead of an interface +and derives every name that VM could occupy, so neither has to be recorded for +teardown to work. The VMM sweeps before preparing a launch as well as on stop, +which makes a launch self-healing regardless of what the record says. + +A bridge prepare also carries two things `netd` does not need to build the TAP. +`workdir` names the VM's directory on the host: untrusted, never read for a +decision, and present only so an operator reading `netd`'s log can get from an +opaque TAP name back to the VM. `ingress` states the host ports that NIC should make +reachable at its guest, which the VMM cannot arrange itself — it runs without +`CAP_NET_ADMIN` by design, and QEMU's `hostfwd=` entries need a user-mode netdev +that a bridge NIC does not have. The `netd` in this repository builds interfaces +and does not forward ports; it says so by leaving `ingress` out of its response, +the same reading `queues` gets, so a caller can tell "this netd does not do that" +from "nothing was asked for" instead of assuming ports were forwarded because a +TAP came back. + ## Deployment modes -Production should run one shared service. `netd` reads only the `[netd]` -section, so its root-owned configuration can be small and independent of every -VMM instance: +Production should run one shared service. `netd` reads the `[netd]` section, +plus `cvm.network_filter.mode` if the file has one, so its root-owned +configuration can be small and independent of every VMM instance: ```toml # /etc/dstack/netd.toml @@ -110,8 +141,33 @@ VMM instance: socket = "/run/dstack/netd.sock" socket_mode = 0o660 libvirt_uri = "qemu:///system" + +# Required here because this file has no [cvm] section for netd to read the +# node's policy from. +[netd.network_filter] +mode = "libvirt" +filter = "clean-traffic" +parameters = {} ``` +`[netd.network_filter]` is netd's own copy of the invariant, not a convenience. +netd is the privileged side of the socket, and anything that can reach the +socket can ask for an unfiltered TAP on a host bridge — a request a filtering +node has to refuse in the daemon rather than in its caller. When netd and the +VMM share one `vmm.toml`, leaving it unset derives it from +`[cvm.network_filter]` so the two cannot drift apart; a malformed section is a +startup error rather than a silent fallback to "filter nothing". + +The request says only *whether* to bind a filter, never which one. A caller that +named the filter could name `allow-arp`, which contains no drop rule at all, or +pin `clean-traffic` to the gateway's MAC and IP through its parameters, and +still satisfy a policy that asked for "some filter". + +A macvtap parent is refused when filtering is required and the parent is a host +bridge or is enslaved to one: nwfilter does not apply to macvtap, so that +request is the same unfiltered access to the same segment, spelled with a +different operation. + Production deployments can use systemd socket activation. The socket unit owns the filesystem mode and ownership; `netd.socket_mode` applies only to the standalone bind path. @@ -162,10 +218,38 @@ sudo dstack-vmm --config ./vmm.toml \ --netd-socket /run/dstack-dev/netd.sock ``` -User networking and bridge networking with `mode = "none"` never connect to -`netd`. Libvirt mode fails closed if `netd` is unavailable. - -Filtered TAP netdevs currently set `vhost=off`. This keeps the initial backend -on the directly bound TAP path and avoids adding `/dev/vhost-net` permissions -to the QEMU user. It is a deliberate security-first throughput tradeoff; a -future configurable vhost mode requires equivalent filter integration tests. +User networking and a caller-supplied netdev never ask `netd` to build an +interface. The VMM still contacts the socket for such a VM -- every launch and +every stop releases whatever the VM held, before it decides whether it needs +anything built -- but nothing about the VM depends on the answer. Bridge and +macvtap do ask, and fail closed if `netd` is unavailable. + +Filtered TAP netdevs follow the node's `vhost` and `queues` settings like any +other TAP-backed NIC (see [network-data-plane.md](network-data-plane.md)). The +nwfilter binding is installed on the host TAP interface, so packets traverse it +whether they were written by QEMU or by a vhost worker; filtering is unaffected +by the data plane choice. Enabling vhost does require the QEMU user to be able +to open `/dev/vhost-net`. + +`netd` creates the TAP for unfiltered bridge NICs too. Those TAPs carry no +nwfilter binding, so a bridge node needs `netd` even when +`network_filter.mode = "none"` — see +[bridge-networking.md](bridge-networking.md) for why the host interface has a +single owner. + +An empty filter name is what selects that unfiltered TAP, so `mode = "libvirt"` +with an empty `filter` is rejected at config load rather than quietly producing +an unbound TAP. + +Removal carries the same distinction: the VMM tells `netd` whether the interface +it is asking about was created with a binding, from a record made when it was +built rather than from configuration that may have changed since. A binding it +was told about must be gone before `netd` returns; otherwise `netd` still asks +libvirt to clear one — an interface name is reused by the same VM, and a +leftover binding's rules would be inherited — but a `libvirtd` it cannot reach +is a warning rather than a failure. So a node with `virsh` installed and no +running `libvirtd` can create and destroy multiqueue TAPs. The flag defaults to +true on the wire, so an older VMM's removals still drop their bindings. + +`netd` requires the `virsh` binary to be present whatever the filter mode; it is +`libvirtd` that unfiltered work does not need. diff --git a/docs/macvtap-networking.md b/docs/macvtap-networking.md index 544408d31..362de4e87 100644 --- a/docs/macvtap-networking.md +++ b/docs/macvtap-networking.md @@ -19,6 +19,9 @@ Configure a NIC through node configuration or an authorized VMM RPC request: `parent` must name an existing host interface. `macvtap_mode` may be `private`, `bridge`, `vepa`, or `passthru`; an empty value selects `private`. +Macvtap NICs also honour the `vhost` and `queues` settings described in +[network-data-plane.md](network-data-plane.md); netd creates the interface with +matching hardware queues and the launcher opens `/dev/tapN` once per queue. The configured netd socket permissions apply in the same way as for libvirt-filtered bridge networking. @@ -49,8 +52,9 @@ and the same deterministic MAC address passed to QEMU. Netd then: 4. reads its kernel-assigned ifindex and waits for `/dev/tap`; and 5. returns that runtime device path to the VMM. -The per-VM launcher opens the character device, places it at the fd referenced -by QEMU's `-netdev tap,fd=...` argument, and then execs QEMU. This keeps device +The per-VM launcher opens the character device once per queue pair, places the +descriptors at the fds referenced by QEMU's `-netdev tap,fd=...` (or `fds=...`) +argument, and then execs QEMU. This keeps device paths out of persistent VM configuration, works with both Supervisor and systemd process managers, and does not pass network fds through `sudo`. diff --git a/docs/network-data-plane.md b/docs/network-data-plane.md new file mode 100644 index 000000000..c80af880f --- /dev/null +++ b/docs/network-data-plane.md @@ -0,0 +1,293 @@ +# virtio-net data plane tuning + +Every CVM NIC has two knobs that decide how many packets it can move: whether +the host kernel's vhost-net data plane is used, and how many virtio-net queue +pairs the device exposes. vhost is set per node and overridable per VM; queue +pairs have no node-wide setting at all, for the reason given under +Configuration. + +## Why it matters + +Without vhost-net, QEMU drains every received packet on its single main-loop +thread. That thread is the ceiling, and it does not grow with vCPUs: + +``` +maximum packets per second ≈ 1 core ÷ per-packet main-loop cost +``` + +The per-packet cost varies with traffic shape — a few microseconds for uniform +synthetic streams, tens of microseconds for bidirectional short-connection +traffic — so the ceiling is a property of the workload, not a fixed number. +What is fixed is the shape of the failure: throughput climbs normally until the +main loop saturates at 100% of one core, then packets are dropped at the TAP +before they ever reach the guest. Guest-side counters stay clean, which makes +the cliff easy to misdiagnose as a network problem. + +Guest-side outbound traffic uses the same thread, so a busy guest pays the +cost twice over. + +`vhost=on` moves that work into the host kernel. That returns a whole core, but +it relocates the ceiling rather than removing +it: packets now arrive faster than a single guest receive queue can drain, and +the drops reappear at a higher rate. More queue pairs is what removes them, +which is why enabling vhost also enables a queue count that follows the VM's +vCPU count — the two travel together. + +vhost is **off by default** and enabled per node (or per VM). Two things make +it an opt-in rather than a default: turning it on changes the virtio-net device +of every bridge/macvtap VM on its next boot, and it requires `/dev/vhost-net` +to be accessible to the account QEMU runs under, which the VMM cannot verify on +the operator's behalf — see [Enabling vhost on a node](#enabling-vhost-on-a-node). + +## Configuration + +```toml +[cvm] +# Ceiling for both the default and what a deployment may request. +max_net_queues = 16 + +[cvm.networking] +mode = "bridge" +bridge = "dstack-br0" +vhost = true +``` + +Queue pairs are not a node setting. With vhost on they default to the VM's vCPU +count, capped at 16, because the useful number follows the VM rather than the +host — the guest driver uses at most one queue pair per vCPU. A deployment +overrides that per VM, up to `max_net_queues`. + +Raising `max_net_queues` above 16 widens what a deployment may ask for without +moving the default's cap, so a larger VM never silently acquires a worse +default. Lowering it below 16 does lower the default too, because a node that +refuses a request for four queue pairs should not hand out sixteen by itself. +The hard ceiling from any source is 64. + +Without vhost the default is a single queue pair. The QEMU main loop drains +every queue on one thread, so extra queues buy little while still costing more +MSI-X vectors and a changed guest device. An explicit +queue count is still honoured without vhost, since that combination is a +deliberate request rather than a default. The two defaults travelling together +also means a node that never sets `vhost` keeps building the device its VMs +have always had. + +A VM overrides either value at deploy time, and `UpdateVm` changes them +afterwards — the new values apply from the VM's next boot: + +```bash +vmm-cli.py deploy --name my-vm --image dstack-0.5.9 --compose app.yaml \ + --net bridge --net-queues 4 +vmm-cli.py deploy --name latency-vm --image dstack-0.5.9 --compose app.yaml \ + --net bridge --net-no-vhost + +# retune an existing VM +vmm-cli.py update --net-queues 8 --net-vhost + +# stop pinning either value and follow the node and the vCPU count again +vmm-cli.py update --net-queues auto +vmm-cli.py update --net-vhost-default + +# stop pinning the backend too, and follow whatever the node runs +vmm-cli.py update --net default +``` + +Every pin has an un-pin. A VM keeps reporting whatever it pinned, and the +deployment RPC accepts a VM's own held values back even after the node's +policy moves, so a read-modify-write update never strands a VM. + +The web UI exposes both per NIC in the deploy and update dialogs, alongside the +networking mode. Both fields are also on `NetworkingConfig` in the deployment +and update RPCs. A request that +sets only `vhost`/`queues` keeps the node's own networking mode, so tuning does +not force a caller to restate — or be allowed to choose — a backend. `queues` is +rejected above the node's `max_net_queues`; `vhost` is not otherwise restricted, +since it only affects the requesting VM. `GetMeta` reports +`networking.max_queues` so a client can present the real bound. + +The data plane settings are recorded only when a deployment asks for them. +Leave one out and it stays owned by the node, so changing `[cvm.networking]` +later — including setting `vhost = false` to roll the whole node back — still +reaches VMs deployed with some other networking override. + +Naming a backend is different: it pins that NIC's *backend*, resolved at +deployment. Its mode, and the bridge or macvtap parent that names it, are fixed +for the life of the VM, so a later edit to those fields in `[cvm.networking]` +does not move it to another segment. Nothing else is pinned — the MAC prefix, +the user-mode subnet and DHCP start, and the macvtap forwarding mode stay node +settings and are re-read at every launch, so editing them changes every VM's +next boot, including its MAC and therefore its DHCP lease. A request that only +tunes pins nothing at all, including the backend it inherited. + +`GetInfo` reports that configuration back, and both `vmm-cli.py update` and the +web UI read it, change one field, and resend the rest. Two things follow. A +request may name a bridge or macvtap parent the node itself configured even when +the allowlists are empty: leaving the field out already yields exactly that +value, so echoing it grants nothing policy was withholding. And an update may +restate whatever its own VM already pinned, so that moving the node's default +out from under a VM does not leave that VM's configuration unsendable. A NIC +that inherited its backend reports an empty mode, which is the same thing it was +deployed with. + +Neither field reaches the CVM's measurement. The measured VM configuration the +VMM controls covers the OS image, the vCPU and memory counts, several QEMU +layout flags, the *number* of NICs, and `mr_config_id`; the queue count and the +vhost state are not part of it, so retuning a NIC does not change app identity +or require an on-chain update. Adding or removing a NIC does: the NIC count +changes the guest's ACPI tables and therefore RTMR0. + +## Enabling vhost on a node + +Setting `vhost = true` in `[cvm.networking]` is a node-wide behaviour change: +every bridge or macvtap VM that has not pinned its own data plane gets a +different virtio-net device on its next boot — `vhost=on`, `mq=on` with +vCPU-scaled queue pairs, and the matching MSI-X vector count. The device is not +measured, so attestation and app identity are unaffected. Before flipping it: + +1. **Verify `/dev/vhost-net` is accessible to the account QEMU runs under.** + It is `root:kvm 0660` on Debian-family hosts, where adding the account to + the `kvm` group suffices, and `root:root 0600` on several others. If the + account lacks access, QEMU exits at launch and every affected VM stops + restarting. The VMM warns at startup when its own access fails, but it + cannot refuse on that basis — QEMU need not share its credentials. + +2. **Restart `netd` before or together with the VMM.** Every bridge and + macvtap NIC is prepared by `netd`, and the VMM checks that `netd` echoes the + queue count it built. An older `netd` fails that check; the launch is rolled back + and fails with the reason in the VMM log, but the VM does not start until + `netd` is upgraded. + +3. **Roll back by setting `vhost = false`.** The node value reaches every VM + that did not pin `vhost` explicitly, from its next boot; a VM that pinned + `vhost = true` keeps it until updated. + +## What each mode supports + +| Mode | netdev | vhost | queues > 1 | +|---|---|---|---| +| `user` | `user,...` | no backend | not supported | +| `bridge` | `tap,ifname=` via netd | yes | yes | +| `macvtap` | `tap,fd=` / `tap,fds=` | yes | yes | +| `custom` | operator's own string | operator's own string | no, not settable | + +QEMU's `bridge` netdev accepts neither `vhost=` nor `queues=`, and the setuid +`qemu-bridge-helper` behind its `tap` netdev returns exactly one descriptor. So +bridge mode runs on a TAP that `netd` creates: persistent, `multi_queue` when +asked for, and opened once per queue by QEMU. That is true of every bridge NIC, +filtered or not, single-queue or not — see +[bridge-networking.md](bridge-networking.md) for why the host interface has one +owner. `netd` requires the `virsh` binary to be installed even when nothing is +filtered, though it does not require a reachable `libvirtd`. One-shot +`dstack-vmm run` does not manage netd interface lifecycle, so it refuses bridge +and macvtap NICs outside `--dry-run`. `netd` reports back the queue count it +created, and the VMM refuses to launch on a mismatch — a `netd` deployed +separately as a root service can be older than the VMM asking it for +multiqueue, and QEMU would otherwise reject the interface from inside the +per-VM launcher. + +For macvtap, the per-VM launcher opens the `/dev/tapN` character device once +per queue pair and hands QEMU the descriptors as `fds=`. `netd` creates the +interface with matching `numtxqueues`/`numrxqueues`. + +Custom mode owns its whole netdev string, including any `vhost=`/`queues=` +options, and its guest device stays single-queue: the VMM cannot edit that +string, so it has no way to make a multiqueue device line agree with it. For the +same reason `GetInfo` reports no vhost state and no queue count for a custom +NIC, rather than asserting the resolved defaults over a string it never read. A +hand-written multiqueue netdev will not pair with a multiqueue guest device +today. + +Naming a backend that cannot carry vhost or a queue count, and then asking for +one, is refused — the request is yours to correct. Inheriting such a backend is +not, because the node chose it and may choose another tomorrow; the request +reads as off, or as one queue pair, until then. + +## Choosing a queue count + +The default suits bandwidth-bound workloads. Latency-sensitive ones should ask +for fewer: more queues spread receive processing over more vCPUs, and under TDX +a cross-vCPU wakeup costs an IPI and a VM exit. Measured on one 8-vCPU TDX CVM, +changing only the guest's channel count: + +| Queue pairs | Short-connection throughput | +|---|---| +| 1 | 22.3k conn/s | +| 2 | ~20k conn/s | +| 4 | 15–21k conn/s | +| 8 | 6.2–7.7k conn/s | + +The same CVM with 8 queues moved 3.0 Mpps of 64-byte UDP with no loss, against +roughly 600k with one queue. The trade is real in both directions, so a VM +serving many short connections should set `--net-queues 1` and measure. + +A VM with fewer vCPUs than queues leaves the extra pairs idle — `ethtool -l +eth0` reports the smaller number. An explicit over-provision is not rejected at +deployment, because `vmm-cli.py resize` can raise the vCPU count later. + +Queue pairs also cost guest memory — each RX ring keeps 256 page-sized buffers +posted, about 1 MB per queue pair plus per-queue NAPI and socket state — but at +any realistic RAM/vCPU shape this is noise. Pushed to a shape no deployment +uses (1 GB of RAM with 16 vCPUs, so 16 queue pairs by default), sustained load +did produce atomic order-0 page-allocation failures in RX refill +(`try_fill_recv` in the guest log); 2 GB at the same shape ran clean. Since the +queue default follows the vCPU count and a VM with that many vCPUs carries far +more memory in practice, this needs no tuning — it is recorded here so the +symptom is searchable. The TDX bounce-buffer pool is not a constraint either: +the guest kernel sizes swiotlb at 6% of RAM clamped to [64 MB, 1 GB] with no +`swiotlb=` parameter, while peak demand is bounded by ring size at about 2 MB +per queue pair — a deliberately undersized 32 MB pool sustained full +multiqueue line rate with zero `swiotlb buffer is full` events. + +`vectors` is derived, never configured: `2N + 2`, one vector per queue +direction plus config and control. One queue pair emits no `mq=on` or +`vectors=` at all, leaving the guest device line byte for byte identical to the +one before this feature. The `-netdev` half carries `vhost=on|off` either way, +since that is what selects the backend. + +## Requirements + +The account running QEMU must be able to open `/dev/vhost-net`, which is +`root:kvm 0660` on a stock host — add that account to the `kvm` group. The +`vhost_net` module autoloads on first open. + +`GetInfo` reports the data plane each interface actually got, so a bridge NIC +that fell back for want of a helper reads as `vhost: false` rather than +advertising something it is not using. For a VM that is not running there is no +interface to describe, so it reports what the next launch would build instead -- +the same calculation, against the node configuration and manifest as they stand +now, rather than the ones a finished boot ran under. + +If that account lacks access, QEMU exits at startup and the VM never boots — +there is no fallback to the userspace backend at this point, on any QEMU +version (verified on 8.2.2 and 10.2). What the per-VM launcher log shows +depends on the version: QEMU 8.2 prints `warning: tap: open vhost char device +failed: Permission denied` (once per queue) and then dies on `net/net.c:1185: +net_client_init1: Assertion 'nc' failed` — an upstream bug +([qemu#1486](https://gitlab.com/qemu-project/qemu/-/issues/1486)); later +versions exit cleanly with `Could not open '/dev/vhost-net'`. Grep for either. +The VMM does not refuse a launch over this: QEMU need not share the VMM's +credentials, so a refusal based on the VMM's own access would block deployments +the host can run. It warns instead — when the device node is missing outright, +and when the VMM's own open is denied, since QEMU usually does share its +account. + +QEMU does have a *runtime* fallback, at a different failure point: once the +netdev initialized with vhost, a later `vhost_net_start()` failure at guest +driver activation logs `unable to start vhost net: : falling back on +userspace virtio` and keeps the NIC working on the userspace data path. That +path is reachable only after `/dev/vhost-net` was opened successfully at +launch, so an access problem never lands there. If it does fire, it is the one +case where `GetInfo` can overstate the data plane — the interface reports the +vhost state the launch settled while the packets take the userspace path — and +that QEMU log line is the indicator. + +vhost-net works normally in a TDX guest: the virtio rings and buffers live in +shared, unencrypted memory precisely so a host-side backend can reach them. +This is the same mechanism behind `vhost-vsock-pci`, which dstack has always +used. + +On host kernels older than 6.4 the vhost worker is a free-standing kernel +thread: it is attached to the owner's cgroups, so `cpu.max` and cgroup +accounting do apply, but it is outside QEMU's thread group and so invisible to +`top -H` and to anything reading `/proc//task`. Since 6.4 it is a +`vhost_task` inside that thread group and shows up everywhere the VM's other +threads do. diff --git a/docs/onboarding.md b/docs/onboarding.md index 0d4488bab..28b216ff2 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -149,7 +149,7 @@ The deploy command: Pass `--vcpu`, `--memory`, or `--disk` to change the app resources before you deploy. -The `--port 8080:80` mapping means `host_port:vm_port` and uses TCP on `127.0.0.1`. The full accepted forms are `vm`, `host:vm`, `proto:host:vm`, and `proto:addr:host:vm`. Use `tcp` or `udp` for `proto`. Fixed host and VM ports must be between 1 and 65535. If you omit the host port, or use `auto` or `0`, `dstack` picks a free localhost port and prints the selected mapping after deploy. +The `--port 8080:80` mapping means `host_port:vm_port` and uses TCP on `127.0.0.1`. The full accepted forms are `vm`, `host:vm`, `proto:host:vm`, and `proto:addr:host:vm`, each optionally suffixed with `@` to name which NIC the traffic enters through (a single-NIC VM never needs it). Use `tcp` or `udp` for `proto`. Fixed host and VM ports must be between 1 and 65535. If you omit the host port, or use `auto` or `0`, `dstack` picks a free localhost port and prints the selected mapping after deploy. Open the app from the host: diff --git a/docs/vmm-cli-user-guide.md b/docs/vmm-cli-user-guide.md index 6a669ca4e..6dddc2ba5 100644 --- a/docs/vmm-cli-user-guide.md +++ b/docs/vmm-cli-user-guide.md @@ -290,6 +290,11 @@ Expose services running in your VM: # Multiple ports --port tcp:8080:80 --port tcp:8443:443 + +# Pin a mapping to one NIC: protocol[:host_address]:host_port:vm_port@ +# Without @ the mapping enters through the first user-mode NIC. A VM +# with no user-mode NIC has no backend that can publish the mapping. +--port tcp:0.0.0.0:8443:443@0 ``` #### GPU Assignment diff --git a/dstack/crates/dstack-cli-core/src/ports.rs b/dstack/crates/dstack-cli-core/src/ports.rs index ae662442a..142be1b16 100644 --- a/dstack/crates/dstack-cli-core/src/ports.rs +++ b/dstack/crates/dstack-cli-core/src/ports.rs @@ -33,7 +33,25 @@ pub fn tcp_port_free(addr: &str, port: u16) -> bool { /// * `:` — tcp, 127.0.0.1 /// * `::` /// * `:::` +/// +/// Any of them may carry a trailing `@` naming which NIC the traffic +/// enters through, as `vmm-cli.py --port` does. Without it the VMM picks: the +/// first user-mode NIC, else the first bridge NIC. A single-NIC VM never needs +/// it. pub fn parse_port(spec: &str) -> Result { + let (spec, nic_index) = match spec.rsplit_once('@') { + // Only digits. `parse()` would take " 1" and "+1" as 1, and a NIC index + // is a position in a list the caller wrote. + Some((rest, nic)) if nic.bytes().all(|byte| byte.is_ascii_digit()) && !nic.is_empty() => ( + rest, + Some( + nic.parse::() + .with_context(|| format!("invalid NIC index in --port '{spec}'"))?, + ), + ), + Some((_, nic)) => bail!("invalid NIC index in --port '{spec}': '{nic}' is not a number"), + None => (spec, None), + }; let parts: Vec<&str> = spec.split(':').collect(); let (proto, addr, host, vm) = match parts.as_slice() { [vm] => ("tcp", "127.0.0.1", "auto", *vm), @@ -58,6 +76,7 @@ pub fn parse_port(spec: &str) -> Result { host_address: addr.to_string(), host_port, vm_port, + nic_index, }) } @@ -90,4 +109,23 @@ mod tests { assert!(parse_port("70000:80").is_err()); assert!(parse_port("8080:0").is_err()); } + + #[test] + fn a_mapping_can_name_the_nic_it_enters_through() { + assert_eq!(parse_port("8080:80").unwrap().nic_index, None); + + let pinned = parse_port("udp:0.0.0.0:7483:51820@1").unwrap(); + assert_eq!(pinned.nic_index, Some(1)); + assert_eq!(pinned.protocol, "udp"); + assert_eq!(pinned.host_address, "0.0.0.0"); + assert_eq!(pinned.host_port, 7483); + assert_eq!(pinned.vm_port, 51820); + + // Unpinned must stay unpinned rather than default to NIC 0: the VMM + // resolves it to the first user-mode NIC, which need not be the first. + assert_eq!(parse_port("8080:80@0").unwrap().nic_index, Some(0)); + for bad in ["8080:80@", "8080:80@ 1", "8080:80@+1", "8080:80@a"] { + assert!(parse_port(bad).is_err(), "{bad} must be refused"); + } + } } diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 9624c3a27..8a70141e9 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -102,7 +102,8 @@ enum Command { /// disk size in GB. #[arg(long, default_value_t = 20)] disk: u32, - /// expose a port: `vm` | `host:vm` | `proto:host:vm` | `proto:addr:host:vm` + /// expose a port: `vm` | `host:vm` | `proto:host:vm` | `proto:addr:host:vm`, + /// each optionally suffixed `@` to name the NIC it enters through /// (host omitted/`auto`/`0` ⇒ a free host port is picked). Repeatable. #[arg(long = "port", value_name = "SPEC")] ports: Vec, diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index 6599bc3cb..b32f9fea0 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -294,6 +294,9 @@ pub(crate) async fn cmd_install(mut o: InstallOpts, release_api_base_url: &str) host_address: "127.0.0.1".into(), host_port: kms_port as u32, vm_port: 8000, + // Unpinned: this deploys the node default topology, which + // is one NIC, and the VMM resolves that itself. + nic_index: None, }], ..Default::default() }; diff --git a/dstack/scripts/setup-bridge.sh b/dstack/scripts/setup-bridge.sh index edcbe35a7..ecc3d2901 100755 --- a/dstack/scripts/setup-bridge.sh +++ b/dstack/scripts/setup-bridge.sh @@ -44,83 +44,6 @@ run_cmd() { fi } -# --- Detect qemu-bridge-helper path --- - -find_bridge_helper() { - local paths=( - /usr/lib/qemu/qemu-bridge-helper - /usr/libexec/qemu-bridge-helper - /usr/local/lib/qemu/qemu-bridge-helper - /usr/local/libexec/qemu-bridge-helper - ) - for p in "${paths[@]}"; do - if [[ -f "$p" ]]; then - echo "$p" - return 0 - fi - done - return 1 -} - -# --- Detect current bridge provider --- - -# Returns "libvirt:" if bridge is managed by a libvirt network, -# "standalone" otherwise. -detect_bridge_provider() { - if command -v virsh &>/dev/null; then - local name br - while read -r name; do - [[ -z "$name" ]] && continue - br=$(virsh net-dumpxml "$name" 2>/dev/null | grep -oP "/dev/null) - fi - echo "standalone" -} - -# --- Check functions --- - -check_bridge_helper() { - echo - bold "qemu-bridge-helper" - local helper - if ! helper=$(find_bridge_helper); then - check_fail "qemu-bridge-helper not found" - check_info "Install QEMU: sudo apt install qemu-system-x86" - return - fi - check_pass "found at $helper" - - if [[ -u "$helper" ]]; then - check_pass "setuid bit is set" - else - check_fail "setuid bit not set" - check_info "Fix: sudo chmod u+s $helper" - fi -} - -check_bridge_conf() { - echo - bold "/etc/qemu/bridge.conf" - local conf="/etc/qemu/bridge.conf" - if [[ ! -f "$conf" ]]; then - check_fail "$conf does not exist" - check_info "Fix: sudo mkdir -p /etc/qemu && echo 'allow $BRIDGE' | sudo tee $conf" - return - fi - check_pass "$conf exists" - - if grep -qE "^allow[[:space:]]+($BRIDGE|all)[[:space:]]*$" "$conf" 2>/dev/null; then - check_pass "bridge '$BRIDGE' is allowed" - else - check_fail "bridge '$BRIDGE' not found in $conf" - check_info "Fix: echo 'allow $BRIDGE' | sudo tee -a $conf" - fi -} - check_bridge_interface() { echo bold "bridge interface: $BRIDGE" @@ -326,33 +249,6 @@ check_forward_rules() { # --- Setup: common --- -setup_bridge_conf() { - echo - bold "Setting up /etc/qemu/bridge.conf" - run_cmd sudo mkdir -p /etc/qemu - if [[ -f /etc/qemu/bridge.conf ]] && grep -qE "^allow[[:space:]]+($BRIDGE|all)" /etc/qemu/bridge.conf 2>/dev/null; then - echo " already configured" - else - run_cmd bash -c "echo 'allow $BRIDGE' | sudo tee -a /etc/qemu/bridge.conf" - fi -} - -setup_bridge_helper() { - echo - bold "Setting up qemu-bridge-helper" - local helper - if ! helper=$(find_bridge_helper); then - echo " $(red 'ERROR'): qemu-bridge-helper not found. Install QEMU first." - return 1 - fi - if [[ -u "$helper" ]]; then - echo " setuid already set on $helper" - else - run_cmd sudo chmod u+s "$helper" - echo " setuid set on $helper" - fi -} - setup_ip_forward() { echo bold "Enabling IP forwarding" @@ -681,8 +577,6 @@ cmd_check() { echo "provider: $(bold 'standalone')" fi - check_bridge_helper - check_bridge_conf check_bridge_interface check_dhcp check_dhcp_firewall @@ -728,8 +622,6 @@ cmd_setup() { $DRY_RUN && echo "dry-run: $(yellow 'yes')" # Common setup - setup_bridge_conf - setup_bridge_helper setup_ip_forward # Mode-specific setup @@ -812,7 +704,9 @@ cmd_destroy() { fi fi - # Remove bridge.conf entry + # Remove the bridge.conf entry an older setup added. netd owns every host + # interface now, so qemu-bridge-helper is not used and the `allow` line is + # a standing grant to attach any local user's TAP to the bridge. local conf="/etc/qemu/bridge.conf" if [[ -f "$conf" ]] && grep -qE "^allow[[:space:]]+${BRIDGE}[[:space:]]*$" "$conf" 2>/dev/null; then echo diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index c035e64d0..62bf381be 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -39,19 +39,34 @@ message VmInfo { repeated GuestEvent events = 14; // Effective network interfaces resolved against node config. repeated NetworkInterfaceStatus interfaces = 15; + // Whether a QEMU process exists for this VM right now. The interfaces above + // are what it built only while this is true; otherwise they are what the next + // launch would build. + bool running = 16; } // Runtime status of a resolved VM network interface. message NetworkInterfaceStatus { - // Product-facing mode: "user", "bridge", or "custom". + // Product-facing mode: "user", "bridge", "macvtap", or "custom". string mode = 1; - // QEMU/network backend shape: "slirp", "tap_bridge", or "custom". + // QEMU/network backend shape: "slirp", "tap_bridge", "macvtap", or "custom". string backend = 2; string mac = 3; // Linux bridge name for tap_bridge backend. optional string bridge_name = 4; // QEMU netdev id, e.g. "net0". optional string netdev_id = 5; + // Effective vhost-net data plane state for this interface. Absent for custom + // mode, where the operator supplies the whole netdev string and the VMM does + // not parse it, so it knows of no data-plane state to report. + optional bool vhost = 6; + // Effective virtio-net queue pairs. Absent for custom mode, for the same + // reason as vhost. + optional uint32 queues = 7; + // Effective macvtap forwarding mode, for macvtap interfaces only. Node + // configuration decides it, so it is reported here with the rest of the + // resolved state rather than on the VM's own NetworkingConfig. + optional string macvtap_mode = 8; } // Structured log or lifecycle event emitted by the guest or runtime. @@ -132,9 +147,16 @@ message NetworkingConfig { string bridge_name = 2; // Parent host interface for macvtap mode. string parent = 3; - // Effective macvtap forwarding mode in responses. Deployment requests must - // leave this empty because the mode is controlled by node configuration. + // Deployment requests must leave this empty: the forwarding mode is node + // configuration, and a VM never pins one. The effective value is reported on + // NetworkInterfaceStatus.macvtap_mode instead. string macvtap_mode = 4; + // Move packet processing into the host kernel vhost-net data plane. Unset + // inherits the node default. User mode has no vhost backend and ignores it. + optional bool vhost = 5; + // virtio-net queue pairs. Unset inherits the node default. Bounded by the + // node's cvm.max_net_queues. + optional uint32 queues = 6; } // Requested GPU layout for a CVM. @@ -160,6 +182,16 @@ message PortMapping { uint32 vm_port = 3; // Host address string host_address = 4; + // Which NIC this mapping's traffic enters through, as an index into + // `networks`. Unset picks the first user-mode NIC, which is where QEMU's + // hostfwd entries have always gone and the only place a host port is + // published from. + // + // A VM with one NIC never needs it. With several there is a choice, and it + // used to be made silently: a bridge NIC for external traffic beside a + // user-mode NIC for management -- the topology multi-NIC was added for -- + // put every published port on the management NIC. + optional uint32 nic_index = 5; } // Partial configuration used when mutating an existing VM. @@ -301,6 +333,13 @@ message NetworkingCapabilities { reserved "forward_service_enabled"; // Default bridge configured in vmm.toml [cvm.networking].bridge. string default_bridge = 4; + // Largest virtio-net queue pair count a deployment request may ask for. + uint32 max_queues = 5; + // Whether the node's own backend runs the vhost-net data plane. A NIC that + // pins neither vhost nor a queue count follows this, and without vhost the + // queue count does not scale with vCPUs -- so a client cannot describe what + // an empty queue field will do without it. + bool default_vhost = 6; } // Aggregated metadata exposed through GetMeta. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 836176d4b..518bae52d 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - config::{Config, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, Protocol}, + config::{Config, Networking, NetworkingMode, NicNetworking, ProcessAnnotation, Protocol}, logrotate, netd::{ self, InterfaceIdentity, PrepareBridgeRequest, PrepareMacvtapRequest, @@ -42,16 +42,22 @@ use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; pub(crate) use network::{ - resolve_networking, resolved_networks, validate_resolved_network, validate_resolved_networks, + filters_bridge_traffic, mode_carries_ingress, needs_netd_interface, resolve_networking, + resolved_networks, settle_vhost, stranded_ingress, validate_resolved_network, + validate_resolved_networks, }; pub use qemu::VmConfig; +// Exported so the RPC layer can assert that everything it reports is +// something it also accepts. +#[cfg(test)] +pub(crate) use vm_info::networking_to_proto; pub use workdir::VmWorkDir; mod host_share; mod id_pool; mod image; mod mr_config; -mod network; +pub(crate) mod network; mod qemu; pub(crate) mod registry; mod vm_info; @@ -87,6 +93,10 @@ pub struct PortMapping { pub protocol: Protocol, pub from: u16, pub to: u16, + /// Which NIC carries this mapping. `None` resolves by the node's rule; see + /// [`crate::app::network::ingress_nic`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nic_index: Option, } /// An extra disk attached to the VM (e.g. a pre-baked verity volume). `source` @@ -126,7 +136,7 @@ pub struct Manifest { #[serde(default)] pub swtpm: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub networks: Vec, + pub networks: Vec, #[serde(default)] pub volumes: Vec, } @@ -300,6 +310,9 @@ pub struct App { state: Arc>, /// Pull status for registry images: tag → status. pub(crate) pull_status: Arc>>, + /// One lock per VM, held across a launch or a teardown. See + /// [`App::launch_lock`]. + launch_locks: Arc>>>>, } const GUEST_AGENT_RPC_TIMEOUT: Duration = Duration::from_secs(30); @@ -327,12 +340,42 @@ impl App { state: Arc::new(Mutex::new(AppState { cid_pool, vms: HashMap::new(), + removing: HashSet::new(), })), config: Arc::new(config), pull_status: Arc::new(Mutex::new(std::collections::HashMap::new())), + launch_locks: Arc::new(Mutex::new(HashMap::new())), } } + /// Serializes everything that creates or deletes one VM's host interfaces. + /// + /// A launch reads whether QEMU is already up and then spends many awaits -- + /// a GPU reset, a whole netd conversation, building the QEMU arguments -- + /// before it launches anything. Nothing used to cover that window. Two + /// `StartVm` calls, or one racing the auto-restart timer, could both read + /// "not running", and the loser's sweep would delete the TAPs the winner's + /// QEMU was already holding open: a live VM silently loses its networking, + /// and the loser's error path then clears the winner's record of it. The + /// authoritative rejection lives in the supervisor, which is reached long + /// after the damage is done. + /// + /// A tokio mutex, because it is held across awaits. Per VM, because a slow + /// start must not stall unrelated ones. Taken by stop and by removal as + /// well as by start: those delete the same interfaces from the other side. + pub(crate) async fn launch_lock(&self, id: &str) -> tokio::sync::OwnedMutexGuard<()> { + self.launch_lock_handle(id).lock_owned().await + } + + fn launch_lock_handle(&self, id: &str) -> Arc> { + let mut locks = self.launch_locks.lock().or_panic("mutex poisoned"); + // A VM that is neither starting nor stopping leaves the map holding + // the only reference, so the map stays the size of what is in flight + // rather than of every VM this process has ever touched. + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + locks.entry(id.to_string()).or_default().clone() + } + pub async fn load_vm( &self, work_dir: impl AsRef, @@ -355,7 +398,7 @@ impl App { let vm_id = manifest.id.clone(); let mut runtime_networks = vm_work_dir.runtime_networks(); if runtime_networks.is_empty() && cids_assigned.contains_key(&vm_id) { - runtime_networks = resolved_networks(&manifest, &self.config.cvm); + runtime_networks = self.inferred_runtime_networks(&manifest); if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); } @@ -396,6 +439,19 @@ impl App { Ok(()) } + /// Refuses an operation on a VM that is being removed. + /// + /// Cheap, and taken before the launch lock as well as under it. Removal + /// holds that lock until the VM has exited -- hours, by its own estimate -- + /// so anything that only asked afterwards would wait the removal out in + /// order to be told no. + pub(crate) fn refuse_if_removing(&self, id: &str) -> Result<()> { + if self.lock().is_removing(id) { + bail!("VM is being removed"); + } + Ok(()) + } + pub async fn start_vm(&self, id: &str) -> Result<()> { self.start_vm_with_restart_policy(id, true).await } @@ -410,13 +466,24 @@ impl App { vm.state.auto_restart.reset(); } } - { - let state = self.lock(); - if let Some(vm) = state.get(id) { - if vm.state.removing { - bail!("VM is being removed"); - } - } + // Before the lock as well as after it. Removal holds the lock until the + // VM has exited, so a launch that only asked afterwards would wait that + // out -- hours, by removal's own estimate -- to be told no. Asking + // first is not sufficient on its own, because the marker can be set + // while this waits; asking again under the lock is what makes it + // authoritative. + self.refuse_if_removing(id)?; + // Everything below reads whether this VM is running and acts on the + // answer for as long as the launch takes. See [`App::launch_lock`]. + let _launch = self.launch_lock(id).await; + self.refuse_if_removing(id)?; + // A restart decided before a stop must not outlive it. The decision + // read `started` from disk; `stop_vm` writes it false under this lock, + // so re-reading it here is what makes the stop stick. An explicit start + // sets the flag itself and has nothing to re-read. + if !reset_restart_policy && !self.work_dir(id)?.started().unwrap_or(false) { + debug!(id, "skipping automatic restart: the VM was stopped"); + return Ok(()); } self.sync_dynamic_config(id)?; let is_running = self @@ -454,7 +521,7 @@ impl App { append_boot_separator(&path); } - let mut runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); + let mut runtime_networks = self.runtime_networks(&vm_config.manifest); let devices = self.try_allocate_gpus(&vm_config.manifest)?; let gpu_host_config = self.config.cvm.gpu.clone(); let devices_to_sanitize = devices.clone(); @@ -463,13 +530,10 @@ impl App { }) .await .context("GPU sanitization task failed")??; - if let Err(error) = self - .prepare_filtered_networks(&vm_config, &mut runtime_networks) - .await - { - let _ = work_dir.clear_runtime_networks(); - return Err(error); - } + // Keep any earlier snapshot on failure: preparation may not have + // been able to persist the cleanup marker for legacy NICs yet. + self.prepare_netd_networks(&vm_config, &mut runtime_networks) + .await?; let processes = match vm_config.config_qemu( &work_dir, &self.config.cvm, @@ -478,16 +542,12 @@ impl App { ) { Ok(processes) => processes, Err(error) => { - let _ = self - .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) - .await; + self.release_vm_interfaces(&vm_config.manifest.id).await; return Err(error); } }; if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) { - let _ = self - .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) - .await; + self.release_vm_interfaces(&vm_config.manifest.id).await; return Err(error); } { @@ -497,17 +557,13 @@ impl App { } for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { - if let Err(cleanup_error) = self - .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) - .await - { - warn!(id, %cleanup_error, "failed to roll back filtered networking"); - } - if let Err(clear_err) = work_dir.clear_runtime_networks() { - warn!( - id, - "failed to clear runtime networks after start failure: {clear_err}" - ); + if self.release_vm_interfaces(&vm_config.manifest.id).await { + if let Err(clear_err) = work_dir.clear_runtime_networks() { + warn!( + id, + "failed to clear runtime networks after start failure: {clear_err}" + ); + } } if let Some(vm_state) = self.lock().get_mut(id) { vm_state.state.runtime_networks.clear(); @@ -532,34 +588,89 @@ impl App { } pub async fn stop_vm(&self, id: &str) -> Result<()> { + // Removal stops the VM itself and holds the launch lock while it does, + // so this would otherwise wait hours to do again what is already being + // done. + self.refuse_if_removing(id)?; if let Some(vm) = self.lock().get_mut(id) { vm.state.auto_restart.reset(); } + // Teardown deletes the same interfaces a launch creates, and derives + // their names rather than reading a record, so it must not overlap one. + let _launch = self.launch_lock(id).await; self.set_started(id, false)?; self.stop_vm_process(id).await?; - let networks = self.work_dir(id)?.runtime_networks(); - self.remove_filtered_networks(id, &networks).await?; + // Not fallible: a VM that has been asked to stop is stopped whether or + // not netd could be reached. What is left behind is reclaimed by this + // VM's next launch, which releases before it prepares, or by its + // removal, which will not finish until the release lands. A VM that is + // never started or removed again keeps its interfaces, and its + // directory is still there to say whose they are. + self.release_vm_interfaces(id).await; Ok(()) } - async fn prepare_filtered_networks( + async fn prepare_netd_networks( &self, vm: &VmConfig, networks: &mut [Networking], ) -> Result<()> { - if self.config.cvm.network_filter.mode == NetworkFilterMode::None - && !networks - .iter() - .any(|network| network.mode == NetworkingMode::Macvtap) + // Before the early return, because a mapping with nowhere to go is a + // property of the resolved topology and not of whether netd is in it. + // Deployment refuses every way of asking for one, so reaching this + // means an edit removed the NIC out from under a mapping that named it. + for mapping in stranded_ingress(&vm.manifest.port_map, networks) { + warn!( + vm_id = %vm.manifest.id, + "port mapping {} {}:{} names NIC {:?}, which this VM no longer has a backend \ + for; it will not be published", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.nic_index, + ); + } + // Whatever an earlier boot left behind: a crash between creating an + // interface and recording it, a NIC this VM no longer has, or a whole + // backend it no longer uses. Prepare replaces the names it is about to + // use, but only those, so an index nothing will claim again is only + // reachable from here. + // + // Before the early return, and not inside it. A VM that has moved from + // a bridge to user-mode networking needs this release precisely because + // it no longer wants an interface, and gating it on wanting one is how + // the interfaces it left behind would become unreachable to every + // later launch. + let vm_workdir = self.work_dir(&vm.manifest.id)?; + if vm_workdir + .runtime_networks() + .iter() + .any(needs_netd_interface) { + // Fail before a user-mode boot can overwrite a legacy snapshot + // when its replacement cleanup marker cannot be persisted. + vm_workdir.mark_network_cleanup_pending()?; + } + self.release_vm_interfaces(&vm.manifest.id).await; + if !networks.iter().any(needs_netd_interface) { return Ok(()); } + vm_workdir.mark_network_cleanup_pending()?; let qemu_uid = Uid::effective().as_raw(); + // Only ever read back out of a log line: netd is told where the VM + // lives so an operator holding an opaque TAP name can reach the VM + // without going through the VMM first. + let workdir = self + .work_dir(&vm.manifest.id) + .map(|dir| dir.path().display().to_string()) + .unwrap_or_default(); + // `port_map` is implemented as QEMU `hostfwd=` entries on a user-mode + // netdev, so a bridge NIC drops every one of them. The VMM cannot + // forward them itself -- it runs without CAP_NET_ADMIN by design -- so + // it states the requirement and lets the node's netd answer it. let mut prepared = Vec::new(); for (nic_index, network) in networks.iter_mut().enumerate() { - if network.mode == NetworkingMode::Bridge - && self.config.cvm.network_filter.mode == NetworkFilterMode::None - { + if !needs_netd_interface(network) { continue; } let identity = InterfaceIdentity { @@ -572,30 +683,41 @@ impl App { &network.mac_prefix_bytes(), nic_index, ); - let request = match network.mode { + let queues = network.queue_pairs(); + let filtered = filters_bridge_traffic(network, &self.config.cvm); + let request = match network.nic.mode { NetworkingMode::Bridge => NetdRequest::PrepareBridge(PrepareBridgeRequest { identity: identity.clone(), - bridge: network.bridge.clone(), + bridge: network.nic.bridge.clone(), mac, qemu_uid, - filter: self.config.cvm.network_filter.filter.clone(), - parameters: self.config.cvm.network_filter.parameters.clone(), + // Which filter, and with what parameters, is netd's to + // decide from its own configuration. An unfiltered TAP is + // only asked for by multiqueue, where the node may not run + // libvirt at all. + filtered, + queues, + workdir: workdir.clone(), }), NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap(PrepareMacvtapRequest { identity: identity.clone(), - parent: network.parent.clone(), + parent: network.nic.parent.clone(), mac, qemu_uid, mode: network.macvtap_mode.clone(), + queues, + workdir: workdir.clone(), }), NetworkingMode::User | NetworkingMode::Custom => continue, }; let response = match netd::request(&self.config.netd.socket, &request).await { Ok(response) => response, Err(error) => { - // The client may have timed out while netd was still finishing - // this Prepare. Remove the in-flight identity first; netd's - // serialized accept loop processes it after Prepare completes. + // The client may have timed out while netd was still + // finishing this Prepare. Remove the in-flight identity + // first: the operation lock makes netd run that removal + // after the Prepare it is undoing, whatever order the two + // connections arrived in. if let Err(cleanup_error) = netd::request( &self.config.netd.socket, &NetdRequest::Remove { @@ -606,69 +728,192 @@ impl App { { warn!(%cleanup_error, "failed to roll back in-flight filtered network"); } - for identity in prepared.into_iter().rev() { - if let Err(cleanup_error) = netd::request( - &self.config.netd.socket, - &NetdRequest::Remove { identity }, - ) - .await - { - warn!(%cleanup_error, "failed to roll back prepared filtered network"); - } - } - return Err(error).context("failed to prepare libvirt-filtered networking"); + self.roll_back_prepared_networks(prepared).await; + // netd's own message is about a socket or a TAP, so + // neither the NIC that asked nor what a node has to install + // to satisfy it appears anywhere in the failure. + let unreachable = netd::is_unreachable(&error); + let mode = network.nic.mode.as_str(); + let error = Err(error).context("failed to prepare netd-managed networking"); + return if unreachable { + error.with_context(|| { + format!( + "interface {nic_index} is {mode}, whose host interface only netd \ + can build; run dstack-vmm netd on this host" + ) + }) + } else if queues > 1 { + error.with_context(|| { + format!("interface {nic_index} asked for {queues} queue pairs") + }) + } else { + error.with_context(|| format!("interface {nic_index} is {mode}")) + }; } }; - if network.mode == NetworkingMode::Macvtap { - network.device = response - .device - .context("netd response omitted macvtap device")?; + prepared.push(identity.clone()); + // Everything below runs after netd already built a host interface, + // so a failure has to unwind the same way a failed Prepare does. + let accepted = (|| { + if network.nic.mode == NetworkingMode::Macvtap { + network.device = response + .device + .clone() + .context("netd response omitted macvtap device")?; + } + // QEMU refuses a TAP whose IFF_MULTI_QUEUE state disagrees with + // its own `queues=`, and reports it from inside the per-VM + // launcher. netd echoes what it built, so a netd too old to + // understand the request fails here, where the reason is + // legible. + if queues > 1 && response.queues != Some(queues) { + bail!( + "netd prepared interface {nic_index} with {} queue pairs instead of \ + {queues}; its version may predate multiqueue support", + response.queues.map_or_else( + || "an unreported number of".to_string(), + |q| q.to_string() + ) + ); + } + Ok(()) + })(); + if let Err(error) = accepted { + self.roll_back_prepared_networks(prepared).await; + return Err(error); } - prepared.push(identity); } Ok(()) } - pub(crate) async fn remove_filtered_networks( - &self, - vm_id: &str, - networks: &[Networking], - ) -> Result<()> { - if self.config.cvm.network_filter.mode == NetworkFilterMode::None - && !networks - .iter() - .any(|network| network.mode == NetworkingMode::Macvtap) - { - return Ok(()); + /// The NICs a VM has now, or would get if it were started. + /// + /// While QEMU is up this is what the launch actually built. Once it is + /// down the snapshot describes a boot that is over: the node configuration + /// and the VM's own manifest can both have changed since, so reporting it + /// would answer a question about the past with the grammar of the present. + /// Predict instead, the same way the next launch will. + fn effective_networks(&self, info: &vm_info::VmInfo) -> Vec { + if info.running && !info.runtime_networks.is_empty() { + return info.runtime_networks.clone(); } - let mut first_error = None; - for (nic_index, network) in networks.iter().enumerate().rev() { - if network.mode == NetworkingMode::Bridge - && self.config.cvm.network_filter.mode == NetworkFilterMode::None + self.merge_networks(&info.manifest) + } + + /// Launch-time view of a VM's NICs: node defaults merged in and the + /// vCPU-scaled queue count made concrete. + pub(crate) fn runtime_networks(&self, manifest: &Manifest) -> Vec { + self.merge_networks(manifest) + } + + /// A running VM whose snapshot is missing, because a VMM that predates the + /// snapshot -- or predates it recording what netd built -- started it. + /// + /// Guessing is all that is left, so guess the way that VMM would have, and + /// then write the guess down. Leaving the marker unset would make every + /// later teardown re-derive it from node configuration that may by then + /// have moved, which is the failure this snapshot exists to prevent. + /// + /// The way *that* VMM would have, not this one: a build old enough to leave + /// no snapshot had no vhost and no multiqueue at all, so whatever this + /// node's defaults say now, the QEMU process actually running was given one + /// queue pair and no vhost. Asking `runtime_networks` would apply today's + /// defaults to a launch that predates them, and the guess is persisted, so + /// it would keep describing that VM wrongly for the life of its boot. + fn inferred_runtime_networks(&self, manifest: &Manifest) -> Vec { + let mut networks = self.merge_networks(manifest); + for network in &mut networks { + network.nic.vhost = Some(false); + network.nic.queues = Some(1); + } + networks + } + + /// The merge itself: node defaults applied, then the data plane settled so + /// that every later stage reads one answer instead of recomputing it. + fn merge_networks(&self, manifest: &Manifest) -> Vec { + let mut resolved = resolved_networks(manifest, &self.config.cvm); + settle_vhost(&mut resolved); + resolved + } + + /// Removes interfaces netd already built for a launch that then failed. + async fn roll_back_prepared_networks(&self, prepared: Vec) { + for identity in prepared.into_iter().rev() { + if let Err(cleanup_error) = + netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity }).await { - continue; + warn!(%cleanup_error, "failed to roll back prepared network interface"); } - if !matches!( - network.mode, - NetworkingMode::Bridge | NetworkingMode::Macvtap - ) { - continue; + } + } + + /// Releases every host interface netd holds for this VM. + /// + /// Unconditional and non-fatal, which is one decision made twice. The + /// release path must not be gated on a predicate that can change under it: + /// `needs_netd_interface` reads the VM's *current* backend, and a VM whose + /// NIC was a bridge when its TAP was built and is a user-mode NIC now would + /// skip the release for interfaces that exist. And a VM must be able to + /// stop when the daemon holding its interfaces cannot be reached, or a + /// netd outage becomes a fleet that cannot be stopped. + /// + /// Returns whether netd is known to hold nothing for this VM any more. + /// A removal reads that to decide whether it may delete the workdir: the + /// directory is what says to try again, and deleting it over a failed + /// release is what strands an interface with nothing left to reach it. + pub(crate) async fn release_vm_interfaces(&self, vm_id: &str) -> bool { + let workdir = match self.work_dir(vm_id) { + Ok(workdir) => workdir, + Err(error) => { + warn!(vm_id, %error, "cannot locate network cleanup state"); + return false; } - let identity = InterfaceIdentity { - instance_id: self.config.cvm.instance_id.clone(), - vm_id: vm_id.to_string(), - nic_index, - }; - if let Err(error) = - netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity }).await - { - first_error.get_or_insert(error); + }; + // Upgrade snapshots written before the marker existed. A failed sweep + // must remain recoverable even if the next boot replaces the snapshot + // with user-mode NICs or an update clears it. + if workdir.runtime_networks().iter().any(needs_netd_interface) { + if let Err(error) = workdir.mark_network_cleanup_pending() { + warn!(vm_id, %error, "cannot persist pending network cleanup"); + return false; } } - if let Some(error) = first_error { - return Err(error).context("failed to remove libvirt-filtered networking"); + // Ask for the release, rather than asking whether it can be asked for. + // A probe first would put a second round trip in front of every stop + // and -- worse -- would make a *busy* netd look like an absent one and + // skip the release entirely. The operation itself cannot be misread + // that way: it succeeds, or it says netd is not there, or netd answers + // with a refusal, and only the last of those has a fallback. + match netd::remove_all( + &self.config.netd.socket, + &self.config.cvm.instance_id, + vm_id, + ) + .await + { + Ok(removed) => { + if let Err(error) = workdir.clear_network_cleanup_pending() { + warn!(vm_id, %error, "cannot clear pending network cleanup"); + return false; + } + if removed > 0 { + info!(vm_id, removed, "released netd-managed interfaces"); + } + true + } + Err(error) if netd::is_unreachable(&error) => { + debug!(vm_id, %error, "no netd to release interfaces from"); + false + } + Err(error) => { + warn!( + vm_id, + "failed to release netd-managed interfaces: {error:#}" + ); + false + } } - Ok(()) } pub(crate) async fn stop_vm_process(&self, id: &str) -> Result<()> { @@ -708,23 +953,32 @@ impl App { pub async fn remove_vm(&self, id: &str) -> Result<()> { { let mut state = self.lock(); - let vm = state.get_mut(id).context("VM not found")?; - if vm.state.removing { + state.get(id).context("VM not found")?; + if !state.start_removing(id) { // Already being removed — idempotent return Ok(()); } - vm.state.removing = true; } + // Clear the in-memory mark if anything below fails before ownership of + // the removal is handed to the background task. In particular, a + // removal that cannot persist its crash-recovery marker must not start: + // without that marker a netd outage could leave interfaces no later + // VMM start knows to release. + let mut mark = RemovalMark::new(self.clone(), id); + // Persist the removing marker so crash recovery can resume let work_dir = self.work_dir(id)?; - if let Err(err) = work_dir.set_removing() { - warn!("failed to write .removing marker for {id}: {err:?}"); - } + work_dir + .set_removing() + .with_context(|| format!("failed to write .removing marker for {id}"))?; // User-initiated removal always deletes the workdir let app = self.clone(); let id = id.to_string(); + // `finish_remove_vm` owns the mark from here. Do not clear it in the + // request task while the background removal is still running. + mark.retain(); tokio::spawn(async move { if let Err(err) = app.finish_remove_vm(&id, true).await { error!("Background cleanup failed for {id}: {err:?}"); @@ -739,13 +993,29 @@ impl App { /// /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup. async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> { + // Ordinary exits clear the mark, including the `?`s below and a panic + // in this task, which `tokio::spawn` would otherwise swallow. The one + // recoverable incomplete outcome explicitly retains it. + let mut mark = RemovalMark::new(self.clone(), id); + // Held across the stop, the wait and the release, not just the release. + // `removing` turns launches away, but a launch that passed that check + // before the marker was set is already inside the lock: it has not + // deployed yet, so the wait below sees nothing running and returns at + // once, and the release then deletes the interfaces of the QEMU that + // launch went on to start. Taking the lock first means the launch + // finishes before removal decides anything, and removal then stops what + // it actually started. + let _launch = self.launch_lock(id).await; // Stop the supervisor process (idempotent if already stopped) if let Err(err) = self.stop_vm_process(id).await { debug!("graceful VM stop during removal failed: {err:?}"); } - // Poll until the process is no longer running, then remove it. - // Some VMs take a long time to stop (e.g. 2+ hours), so we wait indefinitely. + // Poll until the process is no longer running, then remove it. The + // stop above is a SIGKILL, so this is however long the kernel takes to + // tear the VM down -- seconds for a large TD, unbounded for one wedged + // in a device reset. Waiting is still right: what follows deletes the + // interfaces and the workdir it is using. let mut poll_count: u64 = 0; loop { match self.supervisor.info(id).await { @@ -777,25 +1047,42 @@ impl App { } } - let runtime_networks = self.work_dir(id)?.runtime_networks(); - if let Err(error) = self.remove_filtered_networks(id, &runtime_networks).await { - warn!(id, %error, "failed to remove filtered networking during VM removal"); - } + let vm_path = self.work_dir(id)?; + // Read before the release, because the release is what makes it stale. + // A VM that never asked netd for an interface -- user mode, a custom + // netdev, or one that never launched -- has nothing for netd to be + // holding, so an absent netd is not a reason to keep its directory. + let held_interfaces = vm_path.network_cleanup_pending() + || vm_path.runtime_networks().iter().any(needs_netd_interface); + let released = self.release_vm_interfaces(id).await; // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. - let vm_path = self.work_dir(id)?; - if delete_workdir || vm_path.is_removing() { + if !(delete_workdir || vm_path.is_removing()) { if vm_path.path().exists() { - if let Err(err) = fs::remove_dir_all(&vm_path) { - error!("failed to remove VM directory for {id}: {err:?}"); - } + info!( + "VM {id} workdir preserved (orphan cleanup): {}", + vm_path.path().display() + ); } - } else if vm_path.path().exists() { - info!( - "VM {id} workdir preserved (orphan cleanup): {}", - vm_path.path().display() + } else if held_interfaces && !released { + // The `.removing` marker and the directory are what a later boot + // reads to retry this, and `remove_all` is idempotent, so keeping + // them costs one retry and losing them strands every interface + // netd still holds: nothing else on the host can name them. + warn!( + "VM {id} keeps its directory because netd did not release its interfaces; \ + the removal resumes at the next VMM start" ); + // The disk marker says this removal must be retried. Keep the + // matching in-memory state too, so start/update/stop cannot revive + // the VM before that retry happens. + mark.retain(); + return Ok(()); + } else if vm_path.path().exists() { + if let Err(err) = fs::remove_dir_all(&vm_path) { + error!("failed to remove VM directory for {id}: {err:?}"); + } } // Free CID and remove from memory (last step) @@ -815,16 +1102,13 @@ impl App { /// Returns false if a cleanup task is already running for this VM. fn spawn_finish_remove(&self, id: &str) -> bool { { - let mut state = self.lock(); - if let Some(vm) = state.get_mut(id) { - if vm.state.removing { - // Already being cleaned up — skip - return false; - } - vm.state.removing = true; + // Claimed in the set rather than in the entry: an orphaned + // supervisor process has no entry, and that is exactly the case + // where the launch lock is held with nothing turning waiters away. + if !self.lock().start_removing(id) { + // Already being cleaned up — skip + return false; } - // If VM is not in memory (e.g. orphaned supervisor process), no entry to guard - // but we still need to clean up the supervisor process. } let app = self.clone(); let id = id.to_string(); @@ -1059,7 +1343,7 @@ impl App { let already_running = cids_assigned.contains_key(&vm_id); let mut runtime_networks = vm_work_dir.runtime_networks(); if runtime_networks.is_empty() && already_running { - runtime_networks = resolved_networks(&manifest, &self.config.cvm); + runtime_networks = self.inferred_runtime_networks(&manifest); if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); } @@ -1164,7 +1448,8 @@ impl App { .map(|vm| { let work_dir = self.work_dir(&vm.config.manifest.id)?; let info = vm.merged_info(vms.get(&vm.config.manifest.id), &work_dir); - Ok(info.to_pb(&self.config.gateway, &self.config.cvm, request.brief)) + let networks = self.effective_networks(&info); + Ok(info.to_pb(&self.config.gateway, request.brief, &networks)) }) .collect::>>()?; Ok(StatusResponse { @@ -1188,14 +1473,18 @@ impl App { pub async fn vm_info(&self, id: &str) -> Result> { let proc_state = self.supervisor.info(id).await?; - let state = self.lock(); - let Some(vm_state) = state.get(id) else { - return Ok(None); + // Snapshot under the lock, then release it: the global state lock is + // held by every other VM's operations, and describing one VM has no + // business keeping it across the work that follows. + let info = { + let state = self.lock(); + let Some(vm_state) = state.get(id) else { + return Ok(None); + }; + vm_state.merged_info(proc_state.as_ref(), &self.work_dir(id)?) }; - let info = vm_state - .merged_info(proc_state.as_ref(), &self.work_dir(id)?) - .to_pb(&self.config.gateway, &self.config.cvm, false); - Ok(Some(info)) + let networks = self.effective_networks(&info); + Ok(Some(info.to_pb(&self.config.gateway, false, &networks))) } pub(crate) fn vm_event_report(&self, cid: u32, event: &str, body: String) -> Result<()> { @@ -1890,6 +2179,341 @@ mod tests { use super::mr_config::{mr_config_version, MrConfigVersion}; use super::*; + fn test_app() -> App { + use rocket::figment::providers::Format as _; + let config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + App::new(config, SupervisorClient::new("http://127.0.0.1:0")) + } + + fn test_config(netd_socket: &Path, run_path: &Path) -> Config { + use rocket::figment::providers::Format as _; + let mut config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + config.netd.socket = netd_socket.to_path_buf(); + config.cvm.instance_id = "test-instance".to_string(); + config.run_path = run_path.to_path_buf(); + config + } + + fn app_talking_to(netd_socket: &Path) -> App { + let run_path = netd_socket.parent().unwrap_or(Path::new("/nonexistent")); + App::new( + test_config(netd_socket, run_path), + SupervisorClient::new("http://127.0.0.1:0"), + ) + } + + /// A supervisor with no remaining process lets removal exercise the real + /// directory and netd paths without launching a VM. + async fn stopped_supervisor() -> (SupervisorClient, tokio::task::JoinHandle<()>) { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut reader = BufReader::new(&mut stream); + let mut header = String::new(); + loop { + header.clear(); + assert!(reader.read_line(&mut header).await.unwrap() > 0); + if header == "\r\n" { + break; + } + } + let body = serde_json::to_string(&supervisor_client::supervisor::Response::Data( + Option::::None, + )) + .unwrap(); + stream + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body, + ) + .as_bytes(), + ) + .await + .unwrap(); + } + }); + (SupervisorClient::new(&format!("http://{address}")), task) + } + + #[tokio::test] + async fn failed_release_survives_snapshot_replacement_and_removal_retry() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(&dir.path().join("absent-netd.sock"), dir.path()), + supervisor.clone(), + ); + let workdir = app.work_dir("vm-1").unwrap(); + std::fs::create_dir_all(workdir.path()).unwrap(); + // A legacy launch only left the runtime snapshot, without a marker. + let network = Networking { + nic: NicNetworking { + mode: NetworkingMode::Bridge, + ..Default::default() + }, + ..Default::default() + }; + workdir.set_runtime_networks(&[network]).unwrap(); + assert!(!workdir.network_cleanup_pending()); + assert!(!app.release_vm_interfaces("vm-1").await); + assert!(workdir.network_cleanup_pending()); + // A subsequent user-mode boot can replace the snapshot. Cleanup must + // still remember the bridge interfaces from the earlier boot. + workdir + .set_runtime_networks(&[Networking::default()]) + .unwrap(); + workdir.set_removing().unwrap(); + assert!(app.lock().start_removing("vm-1")); + app.finish_remove_vm("vm-1", true).await.unwrap(); + assert!(workdir.path().exists()); + assert!(workdir.is_removing()); + assert!(workdir.network_cleanup_pending()); + assert!(app.refuse_if_removing("vm-1").is_err()); + + // Simulate restarting the VMM after netd has recovered. + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); + let restarted = App::new(test_config(netd.socket(), dir.path()), supervisor); + restarted.finish_remove_vm("vm-1", true).await.unwrap(); + assert!(!workdir.path().exists()); + assert_eq!(netd.operations(), vec!["remove_all"]); + server.abort(); + } + + #[tokio::test] + async fn removal_without_a_snapshot_distinguishes_pending_from_never_prepared() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(&dir.path().join("absent-netd.sock"), dir.path()), + supervisor, + ); + for (id, pending) in [("interrupted-prepare", true), ("user-only", false)] { + let workdir = app.work_dir(id).unwrap(); + std::fs::create_dir_all(workdir.path()).unwrap(); + if pending { + // Preparation wrote this before contacting netd, then crashed + // without ever writing runtime-networks.json. + workdir.mark_network_cleanup_pending().unwrap(); + } + workdir.set_removing().unwrap(); + app.finish_remove_vm(id, true).await.unwrap(); + assert_eq!(workdir.path().exists(), pending); + assert_eq!(workdir.is_removing(), pending); + } + server.abort(); + } + + #[tokio::test] + async fn successful_sweep_clears_the_pending_marker() { + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(netd.socket(), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + let workdir = app.work_dir("vm-1").unwrap(); + std::fs::create_dir_all(workdir.path()).unwrap(); + workdir.mark_network_cleanup_pending().unwrap(); + assert!(app.release_vm_interfaces("vm-1").await); + assert!(!workdir.network_cleanup_pending()); + assert!(workdir.path().exists()); + } + + /// A netd outage must not become a fleet that cannot be stopped. + #[tokio::test] + async fn a_stop_survives_a_netd_that_is_not_there() { + let app = app_talking_to(Path::new("/nonexistent/dstack-netd.sock")); + // Returns rather than propagating: there is no error type here on + // purpose, because there is no caller that should act on one. + app.release_vm_interfaces("vm-1").await; + } + + /// A removal deletes the workdir, and the workdir is the only thing left + /// that says to retry. So the release has to say whether it landed: an + /// answer means netd holds nothing, and anything else -- a refusal, or a + /// netd that is not there to ask -- means it may still. + #[tokio::test] + async fn a_release_says_whether_netd_still_holds_anything() { + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); + let app = app_talking_to(netd.socket()); + assert!(app.release_vm_interfaces("vm-1").await); + + // Refused: netd is up and still holding whatever it had. + let refusing = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); + let app = app_talking_to(refusing.socket()); + assert!(!app.release_vm_interfaces("vm-1").await); + + // Not there to ask. A VM that never asked netd for an interface is + // unaffected -- the removal checks that separately -- but one that did + // must keep its directory so a later start can try again. + let app = app_talking_to(Path::new("/nonexistent/dstack-netd.sock")); + assert!(!app.release_vm_interfaces("vm-1").await); + } + + /// A netd too old for the sweep refuses it, and a refusal is not a reason + /// to fail the stop. What it holds stays until this VM launches again or + /// until an operator names it. The stop still succeeds: a netd outage must + /// not become a fleet that cannot be stopped. + #[tokio::test] + async fn a_netd_that_refuses_the_sweep_does_not_fail_the_stop() { + let netd = netd::testing::FakeNetd::spawn(netd::testing::Behavior::Legacy); + let app = app_talking_to(netd.socket()); + app.release_vm_interfaces("vm-1").await; + assert_eq!( + netd.operations(), + vec!["remove_all"], + "asked once, and not asked about afterwards" + ); + } + + /// One request, no question in front of it. A probe on the hot path is a + /// second round trip whose failure mode is silence: a netd too slow to + /// answer it reads as an absent one, and an absent netd's release is + /// skipped. + #[tokio::test] + async fn a_stop_asks_netd_to_sweep_without_a_question_first() { + let netd = + netd::testing::FakeNetd::spawn(netd::testing::Behavior::handling(&["remove_all"])); + let app = app_talking_to(netd.socket()); + app.release_vm_interfaces("vm-1").await; + + assert_eq!(netd.operations(), vec!["remove_all"]); + let sweep = &netd.seen()[0]; + assert_eq!(sweep["vm_id"], "vm-1"); + assert_eq!(sweep["instance_id"], "test-instance"); + // A sweep names no NIC: reaching the indices the caller can no longer + // name is the entire point. + assert!(sweep.get("nic_index").is_none()); + } + + /// The orphan cleanup runs for an ID that never loaded, so a guard living + /// in the VM entry is not there when it holds the launch lock across the + /// whole teardown. Without the mark, `StartVm` on that ID waits the + /// teardown out with no error and no log. + #[tokio::test] + async fn a_removal_with_no_vm_entry_still_turns_operations_away() { + let app = test_app(); + assert!(app.refuse_if_removing("orphan").is_ok()); + assert!(app.lock().start_removing("orphan")); + assert!( + app.refuse_if_removing("orphan").is_err(), + "an orphan being removed is still a VM being removed" + ); + // And the same removal cannot be started twice. + assert!(!app.lock().start_removing("orphan")); + } + + /// An ordinary early error clears the in-memory mark so the caller can + /// retry rather than leaving a VM permanently inaccessible. + #[tokio::test] + async fn a_removal_that_gives_up_early_does_not_leave_the_vm_marked() { + let app = test_app(); + { + let _mark = RemovalMark::new(app.clone(), "vm-1"); + assert!(app.lock().start_removing("vm-1")); + assert!(app.refuse_if_removing("vm-1").is_err()); + } + assert!( + app.refuse_if_removing("vm-1").is_ok(), + "the mark is cleared however the removal ends" + ); + } + + /// Once a removal deliberately keeps its crash-recovery marker, dropping + /// the guard must not make the VM launchable again in the current process. + #[tokio::test] + async fn a_removal_waiting_for_netd_stays_marked() { + let app = test_app(); + assert!(app.lock().start_removing("vm-1")); + { + let mut mark = RemovalMark::new(app.clone(), "vm-1"); + mark.retain(); + } + assert!( + app.refuse_if_removing("vm-1").is_err(), + "the in-memory state must agree with the retained disk marker" + ); + } + + /// A restart decided before a stop must not outlive it. The restart task + /// reads the started flag off disk and only then queues a launch, which + /// waits for the lock the stop is holding; without a re-read under that + /// lock the launch resurrects a VM the operator was told was stopped. + #[tokio::test] + async fn an_automatic_restart_does_not_outlive_the_stop_it_raced() { + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(Path::new("/nonexistent/netd.sock"), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + let work_dir = app.work_dir("vm-1").unwrap(); + std::fs::create_dir_all(work_dir.path()).unwrap(); + + work_dir.set_started(false).unwrap(); + app.start_vm_with_restart_policy("vm-1", false) + .await + .expect("an automatic restart of a stopped VM does nothing"); + + // The flag is the whole difference: with it set, the same call goes on + // to do the work, and fails here for want of a VM to launch. + work_dir.set_started(true).unwrap(); + assert!(app + .start_vm_with_restart_policy("vm-1", false) + .await + .is_err()); + + // An explicit start sets the flag itself and has nothing to re-read, + // so it is never turned away by one. + work_dir.set_started(false).unwrap(); + assert!(app.start_vm("vm-1").await.is_err()); + } + + /// The window a launch spends between reading "not running" and actually + /// starting QEMU is long -- a GPU reset, a netd conversation -- and the + /// sweep inside it deletes interfaces by deriving their names. Two entrants + /// in that window meant the loser deleting the winner's live TAPs. + #[tokio::test] + async fn one_vm_launches_at_a_time_and_the_lock_map_stays_small() { + let app = test_app(); + let held = app.launch_lock("vm-1").await; + + // A different VM is never blocked by it: a slow start must not stall + // every other launch on the node. + let other = tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-2")).await; + assert!(other.is_ok(), "an unrelated VM must not wait"); + + // The same VM is. + let same = tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-1")).await; + assert!(same.is_err(), "a second entrant must wait for the first"); + + drop(held); + drop(other); + tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-1")) + .await + .expect("the lock is released"); + + // Nothing is in flight now, so the map holds nothing either. + assert!(app.launch_locks.lock().unwrap().len() <= 1); + let _ = app.launch_lock("vm-3").await; + assert!(app.launch_locks.lock().unwrap().len() <= 2); + } + #[test] fn accepts_server_generated_ids() { validate_vm_id(&uuid::Uuid::new_v4().to_string()).unwrap(); @@ -1917,7 +2541,7 @@ mod tests { } use crate::config::{ - load_config_figment, CvmPlatform, Networking, NetworkingMode, TdxAttestationVariantConfig, + load_config_figment, CvmPlatform, NetworkingMode, TdxAttestationVariantConfig, }; use dstack_types::{ TdxImageMeasurement, TdxMrtdCandidates, TdxOsImageMeasurement, @@ -2230,17 +2854,10 @@ mod tests { )); let workdir = VmWorkDir::new(&temp); let mut manifest = test_manifest(1024); - manifest.networks = vec![Networking { + manifest.networks = vec![NicNetworking { mode: NetworkingMode::Bridge, bridge: "dstack-br0".to_string(), - parent: String::new(), - macvtap_mode: String::new(), - device: String::new(), - mac_prefix: String::new(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, - netdev: String::new(), + ..NicNetworking::default() }]; workdir.put_manifest(&manifest)?; @@ -2495,17 +3112,10 @@ mod tests { fn vm_measurement_config_ignores_networking_changes() -> Result<()> { let config = test_tdx_config()?; let mut bridge_manifest = test_manifest(2048); - bridge_manifest.networks = vec![Networking { + bridge_manifest.networks = vec![NicNetworking { mode: NetworkingMode::Bridge, bridge: "dstack-br0".to_string(), - parent: String::new(), - macvtap_mode: String::new(), - device: String::new(), - mac_prefix: "02:aa:bb".to_string(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, - netdev: String::new(), + ..NicNetworking::default() }]; let user_manifest = test_manifest(2048); let image = test_tdx_image(true); @@ -2998,6 +3608,14 @@ impl VmState { pub(crate) struct AppState { cid_pool: IdPool, vms: HashMap, + /// The VMs a removal is currently working on. + /// + /// Separate from `VmState::removing` because the set has to outlive the + /// entry. Orphan cleanup runs for IDs that never loaded into `vms`, and + /// `finish_remove_vm` holds the launch lock across the whole teardown, so + /// a guard that lives in the entry cannot turn away the operation that + /// would otherwise wait that teardown out. + removing: HashSet, } impl AppState { @@ -3020,6 +3638,57 @@ impl AppState { pub fn iter_vms(&self) -> impl Iterator { self.vms.values() } + + /// Claims `id` for a removal. False when one already has it. + fn start_removing(&mut self, id: &str) -> bool { + if let Some(vm) = self.vms.get_mut(id) { + vm.state.removing = true; + } + self.removing.insert(id.to_string()) + } + + fn is_removing(&self, id: &str) -> bool { + self.removing.contains(id) + } +} + +/// Clears the in-flight removal mark however the removal ends. +/// +/// `finish_remove_vm` returns early on more than its happy path, and a mark +/// left behind is not a stale flag: every operation on that VM answers "being +/// removed" from then on, including the removal that would retry. +struct RemovalMark { + app: App, + id: String, + clear_on_drop: bool, +} + +impl RemovalMark { + fn new(app: App, id: &str) -> Self { + Self { + app, + id: id.to_string(), + clear_on_drop: true, + } + } + + /// Leave the removal mark in place after this guard goes out of scope. + fn retain(&mut self) { + self.clear_on_drop = false; + } +} + +impl Drop for RemovalMark { + fn drop(&mut self) { + if !self.clear_on_drop { + return; + } + let mut state = self.app.lock(); + state.removing.remove(&self.id); + if let Some(vm) = state.vms.get_mut(&self.id) { + vm.state.removing = false; + } + } } /// Reject VM ids that would escape `run_path` once joined into a filesystem diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index 1b840154f..b946e3ca1 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -9,58 +9,141 @@ use std::path::Path; use anyhow::{bail, Result}; use sha2::{Digest, Sha256}; -use super::Manifest; -use crate::config::{CvmConfig, Networking, NetworkingMode}; +use super::{Manifest, PortMapping}; +use crate::config::{ + CvmConfig, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, MAX_NET_QUEUES, +}; -pub(crate) fn resolve_networking(networking: &Networking, cfg: &CvmConfig) -> Networking { +/// Node configuration merged with what one NIC pins. +/// +/// The node half is taken wholesale and the NIC half overrides it where it is +/// set. Nothing the node owns -- the macvtap forwarding mode, the MAC prefix, +/// the user-mode network parameters, a custom netdev string -- can be +/// overridden here any more, because a [`NicNetworking`] cannot carry one. +pub(crate) fn resolve_networking( + networking: &NicNetworking, + cfg: &CvmConfig, + vcpu: u32, +) -> Networking { let mut resolved = cfg.networking.clone(); - resolved.mode = networking.mode; - resolved.restrict = cfg.networking.restrict || networking.restrict; + // A deployment that only tuned the data plane never named a backend, so + // the node keeps deciding which one this NIC uses -- including after the + // operator changes it. + resolved.nic.inherit_mode = networking.inherit_mode; + resolved.nic.mode = if networking.inherit_mode { + cfg.networking.nic.mode + } else { + networking.mode + }; + // Runtime state, never inherited from configuration or from a previous + // launch. Interface preparation sets both for the NICs it builds, and a + // node configuration that names either is rejected at startup. + resolved.device.clear(); if !networking.bridge.is_empty() { - resolved.bridge = networking.bridge.clone(); + resolved.nic.bridge = networking.bridge.clone(); } if !networking.parent.is_empty() { - resolved.parent = networking.parent.clone(); - } - if !networking.mac_prefix.is_empty() { - resolved.mac_prefix = networking.mac_prefix.clone(); + resolved.nic.parent = networking.parent.clone(); } - if !networking.net.is_empty() { - resolved.net = networking.net.clone(); - } - if !networking.dhcp_start.is_empty() { - resolved.dhcp_start = networking.dhcp_start.clone(); - } - if !networking.netdev.is_empty() { - resolved.netdev = networking.netdev.clone(); + if networking.vhost.is_some() { + resolved.nic.vhost = networking.vhost; } + // Make the vCPU-scaled default concrete here, so every later stage -- + // netd preparation, the QEMU arguments, and removal after a VMM restart -- + // reads one number instead of recomputing it from a vCPU count it may no + // longer have. + resolved.nic.queues = Some(match networking.queues { + // An explicit count is honoured whatever the data plane: multiqueue + // without vhost is a valid, if unusual, thing to ask for. + Some(queues) => queues, + // Without vhost the QEMU main loop drains every queue on one thread, + // so scaling up buys almost nothing while still costing a netd + // interface, extra vectors, and a changed device. Anyone turning vhost + // off is asking for the old data plane; give them the old shape too. + None if resolved.vhost_enabled() => { + Networking::default_queue_pairs(vcpu, cfg.max_net_queues) + } + None => 1, + }); resolved } pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec { - if manifest.networks.is_empty() { - vec![cfg.networking.clone()] + let node_default = [cfg.networking.nic.clone()]; + let requested = if manifest.networks.is_empty() { + &node_default[..] } else { - manifest - .networks - .iter() - .map(|networking| resolve_networking(networking, cfg)) - .collect() + &manifest.networks[..] + }; + requested + .iter() + .map(|networking| resolve_networking(networking, cfg, manifest.vcpu)) + .collect() +} + +/// Whether netd must pre-create the host interface for this NIC. +/// +/// Every macvtap and every bridge NIC. The alternative was a set of conditions +/// -- libvirt filtering, multiqueue -- under which netd was consulted and +/// outside of which the VMM built the interface some other way. Each of those +/// paths had to answer the same questions again and answer them differently: +/// which netdev QEMU gets, whether vhost is really on, and, once port mappings +/// grew a NIC, where a bridge NIC's host ports go. A host interface has one +/// owner now, and `port_map` on a bridge reaches netd on every node rather than +/// only on the ones that happened to filter or to have scaled their queues. +/// +/// The cost is stated plainly: bridge and macvtap need a netd on the host. User +/// mode and a caller-supplied netdev still need nothing. +pub(crate) fn needs_netd_interface(networking: &Networking) -> bool { + matches!( + networking.nic.mode, + NetworkingMode::Macvtap | NetworkingMode::Bridge + ) +} + +/// Whether this NIC's host interface carries a libvirt nwfilter binding. +/// Macvtap never does, and a bridge NIC only does when the node filters. +pub(crate) fn filters_bridge_traffic(networking: &Networking, cfg: &CvmConfig) -> bool { + networking.nic.mode == NetworkingMode::Bridge + && cfg.network_filter.mode == NetworkFilterMode::Libvirt +} + +/// Makes the data plane concrete on a launch-time NIC list. +/// +/// `vhost` on a freshly resolved entry is still a *request*: `None` means +/// inherit from the node, which can change under a VM that is already running. +/// Settling it once, here, is what lets the QEMU arguments and the reported +/// status read the same value for the life of a boot. +pub(crate) fn settle_vhost(networks: &mut [Networking]) { + for networking in networks.iter_mut() { + networking.nic.vhost = Some(networking.vhost_enabled()); } } pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { - if networking.mode != NetworkingMode::Bridge { + // The vCPU-scaled default is bounded by construction; only an explicit + // request can exceed the hard cap. + if networking + .nic + .queues + .is_some_and(|queues| queues > MAX_NET_QUEUES) + { + bail!("networking queues must not exceed {MAX_NET_QUEUES}"); + } + if networking.nic.mode != NetworkingMode::Bridge { return Ok(()); } - if networking.bridge.is_empty() { + if networking.nic.bridge.is_empty() { bail!("bridge networking requested but no bridge is configured"); } if !Path::new("/sys/class/net") - .join(&networking.bridge) + .join(&networking.nic.bridge) .exists() { - bail!("bridge interface '{}' does not exist", networking.bridge); + bail!( + "bridge interface '{}' does not exist", + networking.nic.bridge + ); } Ok(()) } @@ -72,6 +155,104 @@ pub(crate) fn validate_resolved_networks(networks: &[Networking]) -> Result<()> Ok(()) } +/// Warns when a vhost NIC is about to launch on a host that has no +/// `/dev/vhost-net` at all. +/// +/// QEMU exits when `vhost=on` cannot open the device, and it does so from +/// inside the per-VM launcher where the reason is easy to miss. This puts the +/// remediation in the VMM log instead. +/// +/// Both checks are warnings, never refusals. QEMU is not necessarily this +/// process — an externally started supervisor can run it under another account +/// — so neither answers the question that decides the launch. They are the two +/// cheap statements that catch the two ways this actually goes wrong. +/// +/// The permission check matters because the node's mode is not uniform. It is +/// `root:kvm 0660` on Debian-family hosts, where a VMM in the `kvm` group is +/// fine, and `root:root 0600` on several others — where every bridge VM stops +/// restarting after an upgrade turns vhost on node-wide, with the only +/// explanation buried in a per-VM launcher's QEMU output. Existence alone says +/// nothing about that case, which is the likelier of the two. +pub(crate) fn warn_if_vhost_net_missing(networks: &[Networking]) { + const VHOST_NET: &str = "/dev/vhost-net"; + if !networks.iter().any(Networking::vhost_enabled) { + return; + } + // The node is a kmod static device node, so it is present even before + // vhost_net is loaded; QEMU's open autoloads the module. + if !Path::new(VHOST_NET).exists() { + tracing::warn!( + "{VHOST_NET} is missing; vhost networking will fail to start. load the vhost_net \ + module, or set vhost = false in [cvm.networking]" + ); + return; + } + if let Err(error) = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(VHOST_NET) + { + if error.kind() == std::io::ErrorKind::PermissionDenied { + tracing::warn!( + "{VHOST_NET} is not accessible to this process; if QEMU runs under the same \ + account, vhost networking will fail to start. add that account to the device's \ + group, or set vhost = false in [cvm.networking]" + ); + } + } +} + +/// Which NIC an unpinned port mapping's traffic enters through. +/// +/// The first user-mode NIC, which is where QEMU's `hostfwd=` entries have +/// always gone. There is no second choice: nothing else on this host publishes +/// a port, so a VM without one has nowhere to put a mapping and the launch +/// says so. +pub(crate) fn default_ingress_nic(networks: &[Networking]) -> Option { + networks + .iter() + .position(|network| network.nic.mode == NetworkingMode::User) +} + +/// Whether a NIC of this mode has a mechanism to publish a host port at all. +/// +/// QEMU's `hostfwd=`, and nothing else. A bridge TAP is built by netd, and the +/// netd in this repository does not forward host ports; macvtap bypasses the +/// host bridge; a custom netdev is a string the VMM does not interpret. +pub(crate) fn mode_carries_ingress(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::User) +} + +/// Which NIC a port mapping's traffic enters through. +/// +/// One mapping resolves to at most one NIC, and only a user-mode NIC has a +/// mechanism to carry it, so a pin to any other kind resolves to nothing +/// rather than to a NIC with no path into the guest. +/// +/// `None` is a mapping with nowhere to go: a VM with no user-mode NIC, or one +/// whose NICs changed under a mapping that named one. The launch warns about +/// each of those rather than dropping it in silence. +pub(crate) fn ingress_nic(mapping: &PortMapping, networks: &[Networking]) -> Option { + mapping + .nic_index + .or_else(|| default_ingress_nic(networks)) + .filter(|index| { + networks + .get(*index) + .is_some_and(|network| mode_carries_ingress(network.nic.mode)) + }) +} + +/// Names the mappings that resolve to no NIC, for a launch to warn about. +pub(crate) fn stranded_ingress<'a>( + port_map: &'a [PortMapping], + networks: &'a [Networking], +) -> impl Iterator { + port_map + .iter() + .filter(|mapping| ingress_nic(mapping, networks).is_none()) +} + /// Derives a deterministic, locally administered unicast MAC address. /// /// Index zero preserves the legacy single-NIC derivation. Later interfaces @@ -98,7 +279,241 @@ pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) #[cfg(test)] mod tests { - use super::mac_address_for_vm_index; + use super::{ + default_ingress_nic, ingress_nic, mac_address_for_vm_index, needs_netd_interface, + resolved_networks, settle_vhost, stranded_ingress, validate_resolved_networks, + }; + use crate::app::PortMapping; + use crate::config::Protocol; + use crate::config::{Networking, NetworkingMode, NicNetworking}; + + fn macvtap_network() -> NicNetworking { + NicNetworking { + mode: NetworkingMode::Macvtap, + parent: "eth0".into(), + // Most tests here exercise the vhost data plane, which the + // shipped default leaves off; opt in the way a real node would. + vhost: Some(true), + ..NicNetworking::default() + } + } + + /// The same NIC as a resolved value, for the checks that run against what + /// a launch would see rather than against what a VM pins. + fn macvtap_resolved() -> Networking { + Networking { + nic: macvtap_network(), + ..Networking::default() + } + } + + fn node_config(mode: NetworkingMode) -> crate::config::CvmConfig { + use rocket::figment::providers::Format as _; + let config: crate::config::Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + let mut cvm = config.cvm; + cvm.networking.nic.mode = mode; + cvm.networking.nic.bridge = "br0".into(); + cvm.networking.nic.parent = "eth0".into(); + // The shipped default leaves vhost off; the tests here are about the + // vhost data plane, so this node opts in the way a real one would. + cvm.networking.nic.vhost = Some(true); + cvm + } + + /// A node whose configuration never names `vhost` — the upgrade case, + /// where the toml predates the key entirely. + fn unconfigured_vhost_node(mode: NetworkingMode) -> crate::config::CvmConfig { + let mut cvm = node_config(mode); + cvm.networking.nic.vhost = None; + cvm + } + + fn manifest_with(vcpu: u32, networks: Vec) -> crate::app::Manifest { + let mut manifest: crate::app::Manifest = serde_json::from_value(serde_json::json!({ + "id": "vm-1", "name": "n", "app_id": "a", "vcpu": vcpu, "memory": 2048, + "disk_size": 10, "image": "i", "port_map": [], "created_at_ms": 0, + })) + .unwrap(); + manifest.networks = networks; + manifest + } + + #[test] + fn queue_pairs_default_to_the_vcpu_count_up_to_the_cap() { + let cvm = node_config(NetworkingMode::Bridge); + for (vcpu, want) in [(1, 1), (2, 2), (8, 8), (16, 16), (32, 16), (128, 16)] { + let resolved = resolved_networks(&manifest_with(vcpu, vec![]), &cvm); + assert_eq!( + resolved[0].queue_pairs(), + want, + "vcpu {vcpu} should give {want} queue pairs" + ); + } + } + + /// An upgraded node must keep building the device its VMs have always + /// had: userspace virtio, one queue pair. Both the toml that predates the + /// `vhost` key and the shipped default say so. + #[test] + fn a_node_that_never_asked_for_vhost_keeps_the_old_device_shape() { + let shipped = { + use rocket::figment::providers::Format as _; + let config: crate::config::Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + config.cvm.networking.nic.vhost + }; + for vhost in [None, shipped] { + let mut cvm = unconfigured_vhost_node(NetworkingMode::Bridge); + cvm.networking.nic.vhost = vhost; + let resolved = resolved_networks(&manifest_with(16, vec![]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + } + } + + #[test] + fn turning_vhost_off_also_turns_off_the_multiqueue_default() { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.networking.nic.vhost = Some(false); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + + // Per-VM opt-out does the same thing. + let cvm = node_config(NetworkingMode::Bridge); + let mut asked = cvm.networking.nic.clone(); + asked.vhost = Some(false); + let resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 1); + + // But an explicit queue count is still honoured without vhost. + let mut asked = cvm.networking.nic.clone(); + asked.vhost = Some(false); + asked.queues = Some(4); + let resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 4); + } + + #[test] + fn lowering_the_request_ceiling_also_lowers_the_default() { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.max_net_queues = 2; + let resolved = resolved_networks(&manifest_with(16, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 2); + + // Raising it past the scaling cap widens requests, not the default. + cvm.max_net_queues = 32; + let resolved = resolved_networks(&manifest_with(24, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 16); + } + + /// One owner for a host interface, with no condition attached. What used + /// to decide this -- libvirt filtering, a scaled queue count -- decided it + /// per node, so the same VM definition got a netd-built TAP on one host and + /// a `qemu-bridge-helper` TAP on the next. + #[test] + fn every_bridge_and_macvtap_nic_is_netds_to_build() { + let mut cvm = node_config(NetworkingMode::Bridge); + let mut single = cvm.networking.nic.clone(); + single.queues = Some(1); + let resolved = resolved_networks(&manifest_with(8, vec![single.clone()]), &cvm); + assert!(needs_netd_interface(&resolved[0])); + + // Unfiltered, single queue, no vhost -- the shape that used to need no + // netd at all -- is netd's too. + cvm.networking.nic.vhost = Some(false); + single.vhost = Some(false); + let resolved = resolved_networks(&manifest_with(8, vec![single]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + assert!(needs_netd_interface(&resolved[0])); + + let cvm = node_config(NetworkingMode::Macvtap); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(needs_netd_interface(&resolved[0])); + + // The two backends the VMM builds itself still need nothing. + for mode in [NetworkingMode::User, NetworkingMode::Custom] { + let cvm = node_config(mode); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(!needs_netd_interface(&resolved[0])); + } + } + + #[test] + fn an_explicit_queue_count_survives_resolution() { + let cvm = node_config(NetworkingMode::Bridge); + let mut asked = macvtap_network(); + asked.mode = NetworkingMode::Bridge; + asked.queues = Some(2); + let resolved = resolved_networks(&manifest_with(16, vec![asked]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 2); + } + + #[test] + fn user_mode_stays_single_queue_whatever_the_vcpu_count() { + let cvm = node_config(NetworkingMode::User); + let resolved = resolved_networks(&manifest_with(32, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 1); + } + + #[test] + fn validation_never_depends_on_this_process_reaching_vhost_net() { + // QEMU may run under different credentials, so a NIC that asks for + // vhost must validate on hosts where the VMM itself cannot open the + // device. Both of these hold whether or not /dev/vhost-net exists here. + let mut networking = macvtap_resolved(); + assert!(networking.vhost_enabled()); + validate_resolved_networks(&[networking.clone()]).unwrap(); + + networking.nic.queues = Some(4); + validate_resolved_networks(&[networking]).unwrap(); + } + + #[test] + fn queue_counts_above_the_hard_bound_are_rejected() { + let mut networking = macvtap_resolved(); + networking.nic.queues = Some(super::MAX_NET_QUEUES + 1); + let error = validate_resolved_networks(&[networking]).unwrap_err(); + assert!(error.to_string().contains("must not exceed")); + } + + /// The data plane a NIC actually gets is decided once, at launch, and + /// written into the runtime entry. Recomputing it later would let a report + /// about a running VM change under an operator's edit to node + /// configuration, describing a data plane QEMU is not using. + #[test] + fn settling_vhost_records_what_the_launch_decided() { + let mut cvm = node_config(NetworkingMode::Bridge); + let manifest = manifest_with(8, vec![]); + let mut networks = resolved_networks(&manifest, &cvm); + assert_eq!(networks[0].nic.vhost, Some(true)); + settle_vhost(&mut networks); + assert_eq!(networks[0].nic.vhost, Some(true)); + + // The node turns vhost off under a VM that is already running. The + // entry the launch settled keeps its answer; the next boot gets the + // new one. + cvm.networking.nic.vhost = Some(false); + assert_eq!(networks[0].nic.vhost, Some(true)); + let mut next_boot = resolved_networks(&manifest, &cvm); + settle_vhost(&mut next_boot); + assert_eq!(next_boot[0].nic.vhost, Some(false)); + + // An inherited `None` becomes a decision rather than staying a request. + let mut unset = resolved_networks(&manifest, &cvm); + unset[0].nic.vhost = None; + settle_vhost(&mut unset); + assert_eq!(unset[0].nic.vhost, Some(false)); + } #[test] fn primary_mac_keeps_legacy_derivation_and_later_nics_are_distinct() { @@ -111,4 +526,80 @@ mod tests { "c6:74:2c:65:14:b9" ); } + + fn nic(mode: NetworkingMode) -> Networking { + Networking { + nic: NicNetworking { + mode, + ..NicNetworking::default() + }, + ..Networking::default() + } + } + + fn mapping(host_port: u16, nic_index: Option) -> PortMapping { + PortMapping { + address: "0.0.0.0".parse().unwrap(), + protocol: Protocol::Tcp, + from: host_port, + to: host_port, + nic_index, + } + } + + #[test] + fn an_unpinned_mapping_still_lands_where_hostfwd_always_put_it() { + // Existing VMs must not move. QEMU's `hostfwd=` has always gone to the + // first user-mode NIC, so that stays the answer wherever there is one. + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; + assert_eq!(default_ingress_nic(&networks), Some(1)); + assert_eq!(ingress_nic(&mapping(443, None), &networks), Some(1)); + + // With no user-mode NIC there is nowhere at all. netd builds a bridge + // TAP but does not forward host ports, and macvtap and custom have no + // path either. + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::Bridge)]; + assert_eq!(default_ingress_nic(&networks), None); + + let networks = [nic(NetworkingMode::Macvtap), nic(NetworkingMode::Custom)]; + assert_eq!(default_ingress_nic(&networks), None); + assert_eq!(ingress_nic(&mapping(443, None), &networks), None); + } + + #[test] + fn a_pinned_mapping_goes_where_it_says() { + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; + assert_eq!(ingress_nic(&mapping(443, Some(1)), &networks), Some(1)); + // Out of range resolves to nothing rather than to something arbitrary. + // Deployment refuses it outright; a manifest that lost a NIC lands here. + assert_eq!(ingress_nic(&mapping(443, Some(7)), &networks), None); + } + + /// A pin has to be checked against the backend, not just the count. + /// Naming a macvtap or custom NIC used to resolve to that index and then + /// fall out of every branch that could act on it: no `hostfwd=`, no netd + /// request, and no warning either. + #[test] + fn a_pin_to_a_backend_with_no_ingress_resolves_to_nothing() { + let networks = [ + nic(NetworkingMode::Macvtap), + nic(NetworkingMode::Custom), + nic(NetworkingMode::Bridge), + nic(NetworkingMode::User), + ]; + assert_eq!(ingress_nic(&mapping(443, Some(0)), &networks), None); + assert_eq!(ingress_nic(&mapping(443, Some(1)), &networks), None); + // A bridge TAP is netd's, and netd does not forward host ports. + assert_eq!(ingress_nic(&mapping(443, Some(2)), &networks), None); + assert_eq!(ingress_nic(&mapping(443, Some(3)), &networks), Some(3)); + + // And an unpinned mapping on a VM with nowhere to put it is named, + // rather than counted as delivered. + let networks = [nic(NetworkingMode::Macvtap)]; + let port_map = [mapping(443, None), mapping(8080, Some(0))]; + let stranded: Vec<_> = stranded_ingress(&port_map, &networks) + .map(|mapping| mapping.from) + .collect(); + assert_eq!(stranded, vec![443, 8080]); + } } diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index db2fd39d4..fe420a484 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -9,14 +9,15 @@ use super::{ hugepage_numa_nodes, image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, - network::{mac_address_for_vm_index, validate_resolved_networks}, + network::{ + ingress_nic, mac_address_for_vm_index, validate_resolved_networks, + warn_if_vhost_net_missing, + }, pci_numa_node, round_up, GpuConfig, VmWorkDir, }; use crate::{ app::Manifest, - config::{ - CvmConfig, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, - }, + config::{CvmConfig, CvmPlatform, Networking, NetworkingMode, ProcessAnnotation}, netd::{tap_name, InterfaceIdentity}, vm_launcher::{ChildCommand, LaunchSpec, OpenFile}, }; @@ -167,6 +168,14 @@ fn create_hd( Ok(()) } +fn on_off(enabled: bool) -> &'static str { + if enabled { + "on" + } else { + "off" + } +} + fn virtio_pci_device(device: &str, snp: bool) -> String { if snp { format!("{device},disable-legacy=on,iommu_platform=true") @@ -179,6 +188,34 @@ struct PreparedVolume { source: String, } +/// First descriptor the per-VM launcher may hand to QEMU. Zero through two are +/// the standard streams. +const FIRST_INHERITED_FD: i32 = 3; + +/// Descriptors the launcher opens for each macvtap NIC, one per queue pair. +/// +/// Both the launcher's open list and the `-netdev` arguments derive from this +/// one layout, so they cannot disagree about which descriptor belongs to which +/// NIC. +fn macvtap_fd_layout(networks: &[Networking]) -> Vec> { + let mut next_fd = FIRST_INHERITED_FD; + networks + .iter() + .map(|network| { + if network.nic.mode != NetworkingMode::Macvtap { + return Vec::new(); + } + (0..network.queue_pairs()) + .map(|_| { + let fd = next_fd; + next_fd += 1; + fd + }) + .collect() + }) + .collect() +} + struct PreparedQemuLaunch { workdir: VmWorkDir, platform: CvmPlatform, @@ -209,6 +246,7 @@ impl PreparedQemuLaunch { let platform = cfg.resolved_platform(); let networks = networks.to_vec(); validate_resolved_networks(&networks)?; + warn_if_vhost_net_missing(&networks); let volumes = vm .manifest .volumes @@ -352,7 +390,7 @@ impl VmConfig { let has_macvtap = prepared .networks .iter() - .any(|network| network.mode == NetworkingMode::Macvtap); + .any(|network| network.nic.mode == NetworkingMode::Macvtap); let Some(socket) = prepared.swtpm_socket.as_deref() else { if has_macvtap { return self.wrap_launcher(&prepared, process, None, None); @@ -396,14 +434,17 @@ impl VmConfig { swtpm: Option, swtpm_socket: Option, ) -> Result> { + // Each queue pair is a separate open of the same macvtap character + // device; the kernel attaches one tap queue per open. let open_files = prepared .networks .iter() - .enumerate() - .filter(|(_, network)| network.mode == NetworkingMode::Macvtap) - .map(|(index, network)| OpenFile { - fd: (3 + index) as i32, - path: network.device.clone().into(), + .zip(macvtap_fd_layout(&prepared.networks)) + .flat_map(|(network, fds)| { + fds.into_iter().map(|fd| OpenFile { + fd, + path: network.device.clone().into(), + }) }) .collect(); let spec = LaunchSpec { @@ -589,11 +630,7 @@ impl QemuCommandBuilder<'_> { } fn configure_networking(&self, command: &mut Command) -> Result<()> { - let hostfwd_index = self - .prepared - .networks - .iter() - .position(|networking| networking.mode == NetworkingMode::User); + let macvtap_fds = macvtap_fd_layout(&self.prepared.networks); for (index, networking) in self.prepared.networks.iter().enumerate() { let net_id = format!("net{index}"); let mac = mac_address_for_vm_index( @@ -601,51 +638,68 @@ impl QemuCommandBuilder<'_> { &networking.mac_prefix_bytes(), index, ); - let net_device = virtio_pci_device( - &format!("virtio-net-pci,netdev={net_id},mac={mac}"), - self.is_amd_sev_snp(), - ); - let netdev = match networking.mode { + let queues = networking.queue_pairs(); + let vhost = networking.vhost_enabled(); + let mut device = format!("virtio-net-pci,netdev={net_id},mac={mac}"); + if queues > 1 { + // One vector per queue direction, plus config and control. + device.push_str(&format!(",mq=on,vectors={}", 2 * queues + 2)); + } + let net_device = virtio_pci_device(&device, self.is_amd_sev_snp()); + let netdev = match networking.nic.mode { NetworkingMode::User => { + // The user-mode backend has neither, so both are ignored + // here. A caller who *named* this mode and then asked for + // vhost or more than one queue pair is refused by the RPC; + // one who inherited it is not, and lands here. let mut netdev = format!( "user,id={net_id},net={},dhcpstart={},restrict={}", networking.net, networking.dhcp_start, if networking.restrict { "yes" } else { "no" } ); - if hostfwd_index == Some(index) { - for mapping in &self.vm.manifest.port_map { - netdev.push_str(&format!( - ",hostfwd={}:{}:{}-:{}", - mapping.protocol.as_str(), - mapping.address, - mapping.from, - mapping.to - )); + // Only the mappings that resolve to this NIC. A mapping + // lands on exactly one, and that NIC's backend decides the + // mechanism, so a bridge NIC's ports go to netd instead of + // being claimed here as well. + for mapping in &self.vm.manifest.port_map { + if ingress_nic(mapping, &self.prepared.networks) != Some(index) { + continue; } + netdev.push_str(&format!( + ",hostfwd={}:{}:{}-:{}", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.to + )); } netdev } NetworkingMode::Bridge => { - tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge); - match self.cfg.network_filter.mode { - NetworkFilterMode::None => { - format!("bridge,id={net_id},br={}", networking.bridge) - } - NetworkFilterMode::Libvirt => { - let tap = tap_name(&InterfaceIdentity { - instance_id: self.cfg.instance_id.clone(), - vm_id: self.vm.manifest.id.clone(), - nic_index: index, - }); - // Keep the filtered backend conservative: QEMU - // uses the TAP path on which libvirt installed the - // nwfilter binding instead of opening vhost-net. - format!( - "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost=off" - ) - } + tracing::info!( + "bridge networking: mac={mac} bridge={} vhost={vhost} queues={queues}", + networking.nic.bridge + ); + // netd owns the TAP. It is the one component here with + // CAP_NET_ADMIN, so it is the only one that can bind an + // nwfilter or create a persistent IFF_MULTI_QUEUE device -- + // and having it own every bridge TAP is what keeps a VM's + // networking from depending on which of those a node + // happens to use. + let tap = tap_name(&InterfaceIdentity { + instance_id: self.cfg.instance_id.clone(), + vm_id: self.vm.manifest.id.clone(), + nic_index: index, + }); + let mut netdev = format!( + "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost={}", + on_off(vhost) + ); + if queues > 1 { + netdev.push_str(&format!(",queues={queues}")); } + netdev } NetworkingMode::Custom => { if !networking.netdev.contains(&format!("id={net_id}")) { @@ -659,7 +713,23 @@ impl QemuCommandBuilder<'_> { if networking.device.is_empty() { bail!("macvtap interface {index} has not been prepared by netd"); } - format!("tap,id={net_id},fd={},vhost=off", 3 + index) + let fds = macvtap_fds + .get(index) + .filter(|fds| !fds.is_empty()) + .with_context(|| { + format!("macvtap interface {index} has no launcher descriptors") + })?; + let selector = if fds.len() == 1 { + format!("fd={}", fds[0]) + } else { + let fds = fds + .iter() + .map(|fd| fd.to_string()) + .collect::>() + .join(":"); + format!("fds={fds}") + }; + format!("tap,id={net_id},{selector},vhost={}", on_off(vhost)) } }; command.arg("-netdev").arg(netdev); @@ -1021,14 +1091,14 @@ mod tests { }; use super::{ - amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, - PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, + amd_sev_snp_memory_backend_arg, macvtap_fd_layout, parse_amd_sev_snp_qmp_capabilities, + virtio_pci_device, PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, }; use crate::app::image::{Image, ImageInfo}; use crate::app::{needs_swtpm, GpuConfig, GpuSpec, Manifest, PortMapping, VmVolume, VmWorkDir}; use crate::config::{ - Config, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, Protocol, - DEFAULT_CONFIG, + Config, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, + Protocol, DEFAULT_CONFIG, }; use crate::netd::{tap_name, InterfaceIdentity}; use dstack_types::{KeyProviderKind, TeeVariant}; @@ -1080,8 +1150,9 @@ mod tests { ); } - #[test] - fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { + /// Minimal launch fixture. Nothing it points at has to exist on disk; every + /// test overrides the fields it asserts on. + fn test_launch_fixture() -> (Config, VmConfig, PreparedQemuLaunch) { let mut config: Config = Figment::from(Toml::string(DEFAULT_CONFIG)) .extract() .unwrap(); @@ -1103,6 +1174,7 @@ mod tests { protocol: Protocol::Tcp, from: 18080, to: 8080, + nic_index: None, }], created_at_ms: 0, hugepages: false, @@ -1151,7 +1223,7 @@ mod tests { workdir: PathBuf::from("/does-not-exist/vm-1"), gateway_enabled: false, }; - let mut prepared = PreparedQemuLaunch { + let prepared = PreparedQemuLaunch { workdir: VmWorkDir::new("/does-not-exist/vm-1"), platform: CvmPlatform::Tdx, networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], @@ -1167,6 +1239,172 @@ mod tests { snp_host_data: None, snp_launch_params: None, }; + (config, vm, prepared) + } + + /// Builds the `-netdev`/`-device` pairs for one NIC layout. + fn net_args(config: &Config, networks: Vec) -> Vec { + let (_, vm, mut prepared) = test_launch_fixture(); + prepared.networks = networks; + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + process + .args + .windows(2) + .filter(|args| args[0] == "-netdev" || args[0] == "-device") + .map(|args| args[1].clone()) + .collect() + } + + fn bridge_network(config: &Config) -> Networking { + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::Bridge; + networking.nic.bridge = "br0".into(); + // The vhost tests below are about what an opted-in node builds; the + // shipped default leaves it off. + networking.nic.vhost = Some(true); + networking + } + + #[test] + fn every_bridge_nic_gets_the_netd_tap() { + // Not only the filtered or multiqueue ones. A bridge NIC's host + // interface has one owner, so the netdev QEMU is handed does not + // change with the node's filter mode or its queue count. + let (mut config, ..) = test_launch_fixture(); + config.cvm.instance_id = "vmm-a".into(); + let mut networking = bridge_network(&config); + networking.nic.queues = Some(1); + let args = net_args(&config, vec![networking]); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=on" + ))); + // A single queue pair must keep the historical device line byte for byte. + assert!(args.iter().any( + |arg| arg.starts_with("virtio-net-pci,netdev=net0,mac=") && !arg.contains("mq=on") + )); + } + + #[test] + fn disabling_vhost_keeps_the_netd_tap_and_turns_the_data_plane_off() { + let (mut config, ..) = test_launch_fixture(); + config.cvm.instance_id = "vmm-a".into(); + let mut networking = bridge_network(&config); + networking.nic.vhost = Some(false); + let args = net_args(&config, vec![networking]); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=off" + ))); + } + + #[test] + fn multiqueue_bridge_uses_the_netd_tap_and_derives_vectors() { + let (mut config, ..) = test_launch_fixture(); + config.cvm.instance_id = "vmm-a".into(); + let mut networking = bridge_network(&config); + networking.nic.queues = Some(4); + let args = net_args(&config, vec![networking]); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=on,queues=4" + ))); + // vectors = 2 per queue pair, plus config and control. + assert!(args.iter().any(|arg| arg.contains("mq=on,vectors=10"))); + } + + #[test] + fn macvtap_queues_take_one_inherited_descriptor_each() { + let (config, ..) = test_launch_fixture(); + let mut first = config.cvm.networking.clone(); + first.nic.mode = NetworkingMode::Macvtap; + first.nic.parent = "eth0".into(); + first.nic.vhost = Some(true); + first.device = "/dev/tap7".into(); + first.nic.queues = Some(2); + let mut second = first.clone(); + second.device = "/dev/tap9".into(); + second.nic.queues = Some(3); + + let networks = vec![first, second]; + let args = net_args(&config, networks.clone()); + assert!(args.contains(&"tap,id=net0,fds=3:4,vhost=on".to_string())); + assert!(args.contains(&"tap,id=net1,fds=5:6:7,vhost=on".to_string())); + + // The launcher must open exactly those descriptors, in that order. + let layout = macvtap_fd_layout(&networks); + assert_eq!(layout, vec![vec![3, 4], vec![5, 6, 7]]); + } + + #[test] + fn macvtap_keeps_a_single_fd_argument_for_one_queue() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::Macvtap; + networking.nic.parent = "eth0".into(); + networking.nic.vhost = Some(true); + networking.device = "/dev/tap7".into(); + let args = net_args(&config, vec![networking]); + assert!(args.contains(&"tap,id=net0,fd=3,vhost=on".to_string())); + } + + /// The operator owns a custom netdev string and the VMM cannot edit it, so + /// the generated device line must never claim more queues than that string + /// provides -- QEMU refuses the mismatch, from inside the per-VM launcher + /// where the reason is hard to see. + #[test] + fn custom_netdev_keeps_its_string_and_stays_single_queue() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::Custom; + networking.netdev = "tap,id=net0,ifname=custom0,vhost=on,queues=8".into(); + // Even a queue count that reached the entry some other way is ignored. + networking.nic.queues = Some(8); + let args = net_args(&config, vec![networking]); + assert!(args.contains(&"tap,id=net0,ifname=custom0,vhost=on,queues=8".to_string())); + assert!( + args.iter().all(|arg| !arg.contains("mq=on")), + "custom mode must not generate a multiqueue device line: {args:?}" + ); + } + + #[test] + fn user_mode_ignores_vhost_and_keeps_its_netdev() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::User; + networking.nic.vhost = Some(true); + let args = net_args(&config, vec![networking]); + assert!(args + .iter() + .any(|arg| arg.starts_with("user,id=net0,") && !arg.contains("vhost"))); + assert!(args.iter().any( + |arg| arg.starts_with("virtio-net-pci,netdev=net0,mac=") && !arg.contains("mq=on") + )); + } + + #[test] + fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { + let (mut config, vm, mut prepared) = test_launch_fixture(); let process = QemuCommandBuilder { vm: &vm, @@ -1232,22 +1470,10 @@ mod tests { .any(|arg| arg.contains("virtio-net-pci,netdev=net1"))); for network in &mut prepared.networks { - network.mode = NetworkingMode::Bridge; - network.bridge = "br0".into(); - } - let process = QemuCommandBuilder { - vm: &vm, - cfg: &config.cvm, - gpus: &GpuConfig::default(), - prepared: &prepared, + network.nic.mode = NetworkingMode::Bridge; + network.nic.bridge = "br0".into(); + network.nic.vhost = Some(false); } - .build() - .unwrap(); - assert!(process - .args - .iter() - .any(|arg| arg == "bridge,id=net0,br=br0")); - config.cvm.instance_id = "vmm-a".into(); config.cvm.network_filter.mode = NetworkFilterMode::Libvirt; let process = QemuCommandBuilder { @@ -1266,6 +1492,10 @@ mod tests { assert!(process.args.iter().any(|arg| { arg == &format!("tap,id=net0,ifname={expected_tap},script=no,downscript=no,vhost=off") })); + assert!(process + .args + .iter() + .all(|arg| !arg.contains("mq=on") && !arg.contains("vectors="))); prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock")); let process = QemuCommandBuilder { @@ -1294,16 +1524,12 @@ mod tests { prepared.swtpm_socket = None; prepared.networks = vec![Networking { - mode: NetworkingMode::Custom, - bridge: String::new(), - parent: String::new(), - macvtap_mode: String::new(), - device: String::new(), - mac_prefix: String::new(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, + nic: NicNetworking { + mode: NetworkingMode::Custom, + ..NicNetworking::default() + }, netdev: "tap,id=wrong".into(), + ..Networking::default() }]; let error = QemuCommandBuilder { vm: &vm, diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index eab200244..48af6f864 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -11,16 +11,16 @@ use dstack_vmm_rpc as pb; use fs_err as fs; use supervisor_client::supervisor::ProcessInfo; -use super::{ - network::{mac_address_for_vm_index, resolved_networks}, - Manifest, VmState, VmWorkDir, -}; -use crate::config::{CvmConfig, GatewayConfig, Networking, NetworkingMode}; +use super::{network::mac_address_for_vm_index, Manifest, VmState, VmWorkDir}; +use crate::config::{GatewayConfig, Networking, NetworkingMode, NicNetworking}; pub(crate) struct VmInfo { pub manifest: Manifest, pub workdir: PathBuf, pub status: &'static str, + /// Whether a QEMU process exists for this VM right now. The NICs it built + /// are real only while it does. + pub running: bool, pub uptime: String, pub exited_at: Option, pub instance_id: Option, @@ -33,15 +33,6 @@ pub(crate) struct VmInfo { pub runtime_networks: Vec, } -fn networking_mode_name(mode: NetworkingMode) -> &'static str { - match mode { - NetworkingMode::Bridge => "bridge", - NetworkingMode::User => "user", - NetworkingMode::Custom => "custom", - NetworkingMode::Macvtap => "macvtap", - } -} - fn networking_backend_name(mode: NetworkingMode) -> &'static str { match mode { NetworkingMode::Bridge => "tap_bridge", @@ -51,16 +42,83 @@ fn networking_backend_name(mode: NetworkingMode) -> &'static str { } } -fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { +/// The resolved NICs a launch built, or would build, as the RPC reports them. +fn interfaces_to_proto( + vm_id: &str, + effective_networks: &[Networking], +) -> Vec { + effective_networks + .iter() + .enumerate() + .map(|(index, networking)| { + let mac = mac_address_for_vm_index(vm_id, &networking.mac_prefix_bytes(), index); + pb::NetworkInterfaceStatus { + mode: networking.nic.mode.as_str().into(), + backend: networking_backend_name(networking.nic.mode).into(), + mac, + bridge_name: (networking.nic.mode == NetworkingMode::Bridge) + .then(|| networking.nic.bridge.clone()), + netdev_id: Some(format!("net{index}")), + // Custom mode hands the operator the whole netdev string and the + // VMM never parses it, so it has no data-plane state to report. + // Reporting the resolved fields anyway would assert "vhost: off, + // queues: 1" over a netdev the operator may have written with + // `vhost=on,queues=8`. + // + // Otherwise: settled at launch, so an entry carrying no decision + // was written before this VMM recorded one -- by a build that + // had no vhost at all, which is what it should read as. + // Recomputing here instead would let an edit to node + // configuration change what a running VM is said to use. + vhost: (networking.nic.mode != NetworkingMode::Custom) + .then(|| networking.nic.vhost.is_some() && networking.vhost_enabled()), + queues: (networking.nic.mode != NetworkingMode::Custom) + .then(|| networking.queue_pairs()), + // Node-decided, so it belongs with the rest of the resolved + // state. The VM's own record cannot carry one. + macvtap_mode: (networking.nic.mode == NetworkingMode::Macvtap) + .then(|| networking.macvtap_mode.clone()), + } + }) + .collect() +} + +pub(crate) fn networking_to_proto(networking: &NicNetworking) -> pb::NetworkingConfig { + // An entry that inherited its backend reports no mode, so it must report + // none of the fields that only make sense alongside one: a mode-less + // override carrying, say, a parent is something the deployment RPC + // rejects, which would strand the VM's tuning as uneditable. + let pins_backend = !networking.inherit_mode; pb::NetworkingConfig { - mode: networking_mode_name(networking.mode).into(), - bridge_name: if networking.mode == NetworkingMode::Bridge { + // An entry that only tuned the data plane named no backend, and the + // deployment RPC spells that as an empty mode. Reporting the node's + // current mode here would turn a read-modify-write into a request to + // pin it -- which policy may not even permit the caller to make. + mode: if networking.inherit_mode { + String::new() + } else { + networking.mode.as_str().into() + }, + bridge_name: if pins_backend && networking.mode == NetworkingMode::Bridge { networking.bridge.clone() } else { String::new() }, - parent: networking.parent.clone(), - macvtap_mode: networking.macvtap_mode.clone(), + // Scope the macvtap fields to macvtap, the way bridge_name is scoped to + // bridge. Reporting an inherited parent on a bridge NIC produced a + // configuration that could be read but not sent back: the deployment + // RPC rejects `parent` outside macvtap mode. + parent: if pins_backend && networking.mode == NetworkingMode::Macvtap { + networking.parent.clone() + } else { + String::new() + }, + // The forwarding mode is node-controlled, so a VM never pins one and + // the type it stores can no longer carry one. It stays on the wire + // because the deployment RPC still has to reject a caller that sets it. + macvtap_mode: String::new(), + vhost: networking.vhost, + queues: networking.queues, } } @@ -69,15 +127,20 @@ fn sanitize_optional>(value: Option) -> Option { } impl VmInfo { - pub fn effective_networks(&self, cvm: &CvmConfig) -> Vec { - if self.runtime_networks.is_empty() { - resolved_networks(&self.manifest, cvm) - } else { - self.runtime_networks.clone() - } - } - - pub fn to_pb(&self, gateway: &GatewayConfig, cvm: &CvmConfig, brief: bool) -> pb::VmInfo { + /// Takes no `CvmConfig` on purpose. Everything it reports about a VM's + /// data plane was decided when that VM launched and written into + /// `effective_networks`; consulting node configuration here is what let an + /// operator's edit change what a running VM was said to be using. + /// + /// `effective_networks` is passed in rather than derived for the same + /// reason, plus one more: a stopped VM's NICs are a prediction, and only + /// the caller can consult netd to make the prediction its launch would. + pub fn to_pb( + &self, + gateway: &GatewayConfig, + brief: bool, + effective_networks: &[Networking], + ) -> pb::VmInfo { let workdir = VmWorkDir::new(&self.workdir); let vm_config = workdir.manifest(); let custom_gateway_urls = vm_config @@ -91,30 +154,17 @@ impl VmInfo { .map(networking_to_proto) .collect::>(); let configured_networking = configured_networks.first().cloned(); - let interfaces = self - .effective_networks(cvm) - .iter() - .enumerate() - .map(|(index, networking)| { - let mac = mac_address_for_vm_index( - &self.manifest.id, - &networking.mac_prefix_bytes(), - index, - ); - pb::NetworkInterfaceStatus { - mode: networking_mode_name(networking.mode).into(), - backend: networking_backend_name(networking.mode).into(), - mac, - bridge_name: (networking.mode == NetworkingMode::Bridge) - .then(|| networking.bridge.clone()), - netdev_id: Some(format!("net{index}")), - } - }) - .collect(); + let interfaces = interfaces_to_proto(&self.manifest.id, effective_networks); pb::VmInfo { id: self.manifest.id.clone(), name: self.manifest.name.clone(), status: self.status.into(), + // The one predicate that says whether `interfaces` above is what a + // process built or what the next launch would build. Clients used to + // re-derive it from `status`, which answers a different question: + // a VM being removed with QEMU still up is not "running" by that + // string, yet its NICs are real. + running: self.running, uptime: self.uptime.clone(), boot_progress: self.boot_progress.clone(), boot_error: self.boot_error.clone(), @@ -148,6 +198,7 @@ impl VmInfo { .port_map .iter() .map(|mapping| pb::PortMapping { + nic_index: mapping.nic_index.map(|index| index as u32), protocol: mapping.protocol.as_str().into(), host_address: mapping.address.to_string(), host_port: mapping.from as u32, @@ -261,6 +312,7 @@ impl VmState { workdir: workdir.path().to_path_buf(), instance_id, status, + running: is_running, uptime, exited_at: Some(exited_at), boot_progress: self.state.boot_progress.clone(), @@ -276,7 +328,68 @@ impl VmState { #[cfg(test)] mod tests { - use super::sanitize_optional; + use super::{interfaces_to_proto, networking_to_proto, sanitize_optional}; + use crate::config::{NetworkingMode, NicNetworking}; + + /// Custom mode hands the operator the whole netdev string and the VMM never + /// parses it, so it has no data-plane state to report. Reporting the + /// resolved defaults instead asserted "vhost off, one queue" over a netdev + /// the operator may well have written as `vhost=on,queues=8`. + #[test] + fn a_custom_netdev_reports_no_data_plane_rather_than_the_wrong_one() { + use crate::config::Networking; + + let custom = Networking { + nic: NicNetworking { + mode: NetworkingMode::Custom, + ..NicNetworking::default() + }, + netdev: "tap,id=net0,ifname=custom0,vhost=on,queues=8".into(), + ..Networking::default() + }; + let interfaces = interfaces_to_proto("vm-1", &[custom]); + assert_eq!(interfaces[0].backend, "custom"); + assert_eq!(interfaces[0].vhost, None); + assert_eq!(interfaces[0].queues, None); + + // Every other backend still answers the question. + let bridge = Networking { + nic: NicNetworking { + mode: NetworkingMode::Bridge, + vhost: Some(true), + queues: Some(4), + ..NicNetworking::default() + }, + ..Networking::default() + }; + let interfaces = interfaces_to_proto("vm-1", &[bridge]); + assert_eq!(interfaces[0].vhost, Some(true)); + assert_eq!(interfaces[0].queues, Some(4)); + } + + #[test] + fn a_reported_interface_can_be_sent_back_unchanged() { + // GetInfo output feeds UpdateVm, so anything it reports has to satisfy + // the deployment RPC's own validation. What a VM stores can no longer + // carry a node-owned field at all -- `parent` here belongs to bridge + // mode's own entry only because the type still allows both backends' + // identity fields, and reporting still scopes it to the owning mode. + let networking = NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + parent: "eth0".into(), + vhost: Some(false), + queues: Some(2), + ..NicNetworking::default() + }; + let proto = networking_to_proto(&networking); + assert_eq!(proto.mode, "bridge"); + assert_eq!(proto.bridge_name, "br0"); + assert!(proto.parent.is_empty()); + assert!(proto.macvtap_mode.is_empty()); + assert_eq!(proto.vhost, Some(false)); + assert_eq!(proto.queues, Some(2)); + } #[test] fn sanitize_optional_filters_empty_owned_values() { diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index 0b358229f..5809f73ba 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -143,6 +143,26 @@ impl VmWorkDir { self.workdir.join("runtime-networks.json") } + /// Written before asking netd to create anything, and removed only after + /// a successful whole-VM sweep. Unlike the runtime snapshot, this survives + /// failed launches and changes to a topology that no longer uses netd. + pub fn mark_network_cleanup_pending(&self) -> Result<()> { + safe_write::safe_write(self.workdir.join(".netd-pending"), b"") + .context("failed to persist pending network cleanup") + } + + pub fn network_cleanup_pending(&self) -> bool { + self.workdir.join(".netd-pending").exists() + } + + pub fn clear_network_cleanup_pending(&self) -> Result<()> { + match fs::remove_file(self.workdir.join(".netd-pending")) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).context("failed to clear pending network cleanup"), + } + } + pub fn runtime_networks(&self) -> Vec { fs::read_to_string(self.runtime_networks_path()) .ok() @@ -303,7 +323,7 @@ mod tests { let persisted = workdir.runtime_networks(); assert_eq!(persisted.len(), 1); - assert_eq!(persisted[0].parent, "br0"); + assert_eq!(persisted[0].nic.parent, "br0"); assert!(persisted[0].device.is_empty()); assert!(!fs::read_to_string(workdir.runtime_networks_path())?.contains("/dev/tap42")); fs::remove_dir_all(temp)?; diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 2e82e67c6..9ec20a115 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -357,7 +357,6 @@ pub struct CvmConfig { pub qemu_pci_hole64_size: u64, /// QEMU hotplug_off pub qemu_hotplug_off: bool, - /// TDX attestation/hash scheme policy. `legacy` keeps the existing /// digest.txt measurement path; `lite` opts into split measurement CBOR; /// `auto` selects `legacy` for @@ -384,6 +383,12 @@ pub struct CvmConfig { #[serde(default)] pub allowed_macvtap_parents: Vec, + /// Largest virtio-net queue pair count a deployment RPC caller may request. + /// There is no node-wide count for it to bind; lowering it below the + /// scaling cap does lower the vCPU-scaled default too. + #[serde(default = "default_max_net_queues")] + pub max_net_queues: u32, + /// Optional host-side filtering for bridge interfaces. This filter does /// not apply to macvtap interfaces. #[serde(default)] @@ -592,6 +597,13 @@ pub struct NetworkFilterConfig { pub parameters: BTreeMap, } +impl NetworkFilterConfig { + /// Whether every bridge TAP on this node must carry an nwfilter binding. + pub fn requires_binding(&self) -> bool { + self.mode == NetworkFilterMode::Libvirt + } +} + impl Default for NetworkFilterConfig { fn default() -> Self { Self { @@ -615,6 +627,23 @@ pub struct NetdConfig { pub socket_mode: u32, #[serde(default = "default_libvirt_uri")] pub libvirt_uri: String, + /// The bridge filtering policy netd enforces and applies: whether a binding + /// is required, which nwfilter it names, and with what parameters. + /// + /// netd holds this itself rather than taking it from each request. It is + /// the privileged side of the socket, and a caller that chose the filter + /// could name one that drops nothing -- `allow-arp` has no drop rule at all + /// -- or pin `clean-traffic` to the gateway's MAC and IP through its + /// parameters, and still satisfy a policy that only asked for "some + /// filter". + /// + /// Unset derives it from `cvm.network_filter` in the same file, which is + /// the whole answer whenever netd and the VMM share one `vmm.toml` -- the + /// normal deployment. Set it explicitly when netd runs with a config that + /// carries no `[cvm]` section, so the daemon holding the privilege is never + /// left inferring policy from a file that does not state it. + #[serde(default)] + pub network_filter: Option, } impl Default for NetdConfig { @@ -623,11 +652,21 @@ impl Default for NetdConfig { socket: default_netd_socket(), socket_mode: 0o660, libvirt_uri: default_libvirt_uri(), + network_filter: None, } } } impl NetdConfig { + /// Resolved policy. Unset means the config named no `[cvm]` section to + /// derive it from, and an unfiltered node is the historical shape. + pub fn filter_policy(&self) -> &NetworkFilterConfig { + static UNFILTERED: std::sync::OnceLock = std::sync::OnceLock::new(); + self.network_filter + .as_ref() + .unwrap_or_else(|| UNFILTERED.get_or_init(NetworkFilterConfig::default)) + } + pub fn validate(&self) -> Result<()> { anyhow::ensure!( self.socket_mode & !0o777 == 0, @@ -715,6 +754,19 @@ impl Config { } validate_networking(&self.cvm.networking)?; + // netd creates an unfiltered TAP when the filter name is empty, which + // is what unfiltered multiqueue bridges need. Libvirt mode must never + // reach that path: it would silently produce an unbound TAP where the + // operator asked for a filtered one. + anyhow::ensure!( + self.cvm.network_filter.mode != NetworkFilterMode::Libvirt + || !self.cvm.network_filter.filter.trim().is_empty(), + "cvm.network_filter.filter must not be empty when mode is libvirt" + ); + anyhow::ensure!( + (1..=MAX_NET_QUEUES).contains(&self.cvm.max_net_queues), + "cvm.max_net_queues must be between 1 and {MAX_NET_QUEUES}" + ); anyhow::ensure!( !self .cvm @@ -828,9 +880,24 @@ fn validate_networking(networking: &Networking) -> Result<()> { "cvm.networking.mac_prefix must contain 1 to 3 two-digit hexadecimal bytes" ); } - match networking.mode { + anyhow::ensure!( + networking.nic.queues.is_none(), + "cvm.networking.queues is not a node setting; queue pairs follow each VM's vCPU count, \ + bounded by cvm.max_net_queues, and a deployment overrides them per NIC" + ); + // Both describe a per-VM entry's relationship to this configuration, so + // neither means anything on the node default itself. + anyhow::ensure!( + !networking.nic.inherit_mode, + "cvm.networking.inherit_mode is per-deployment state and cannot be set on the node default" + ); + anyhow::ensure!( + networking.device.is_empty(), + "cvm.networking.device is runtime state and cannot be set in configuration" + ); + match networking.nic.mode { NetworkingMode::Bridge => anyhow::ensure!( - !networking.bridge.trim().is_empty(), + !networking.nic.bridge.trim().is_empty(), "cvm.networking.bridge must not be empty in bridge mode" ), NetworkingMode::Custom => anyhow::ensure!( @@ -839,7 +906,7 @@ fn validate_networking(networking: &Networking) -> Result<()> { ), NetworkingMode::Macvtap => { anyhow::ensure!( - !networking.parent.trim().is_empty(), + !networking.nic.parent.trim().is_empty(), "cvm.networking.parent must not be empty in macvtap mode" ); anyhow::ensure!( @@ -850,6 +917,7 @@ fn validate_networking(networking: &Networking) -> Result<()> { "cvm.networking.macvtap_mode must be private, bridge, vepa, or passthru" ); } + // User mode has no identity fields of its own to check. NetworkingMode::User => {} } Ok(()) @@ -859,20 +927,62 @@ fn default_allowed_network_modes() -> Vec { vec![NetworkingMode::User, NetworkingMode::Bridge] } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +fn default_max_net_queues() -> u32 { + DEFAULT_MAX_NET_QUEUES +} + +/// Where the vCPU-scaled default stops growing. Each queue pair costs a host +/// vhost thread and two MSI-X vectors, and cross-vCPU wakeups are expensive +/// under TDX, so the benefit runs out well before a large VM's vCPU count. +/// Raising `cvm.max_net_queues` lets a deployment ask for more; it does not +/// move this, because a bigger VM should not silently get a worse default. +pub const DEFAULT_QUEUE_SCALING_CAP: u32 = 16; + +/// Default ceiling on what a deployment RPC caller may request. +pub const DEFAULT_MAX_NET_QUEUES: u32 = 16; + +/// Hard bound on queue pairs from any source, well below anything QEMU or the +/// guest driver would refuse. It exists so a malformed or hostile request +/// cannot ask the host kernel for an unbounded device, not because 64 is a +/// property of virtio-net. +pub const MAX_NET_QUEUES: u32 = 64; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "lowercase")] pub enum NetworkingMode { + /// The backend that needs nothing from the host, so it is what a NIC that + /// names none falls back to. + #[default] User, Bridge, Custom, Macvtap, } -/// Flat networking configuration. The `mode` field selects which backend is -/// active; the remaining fields are only relevant for their respective mode -/// and carry serde defaults so they can be omitted in the config file. -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct Networking { +impl NetworkingMode { + /// The name this mode is written as in `vmm.toml`, in the RPC, and in + /// anything an operator reads. + pub fn as_str(self) -> &'static str { + match self { + NetworkingMode::User => "user", + NetworkingMode::Bridge => "bridge", + NetworkingMode::Custom => "custom", + NetworkingMode::Macvtap => "macvtap", + } + } +} + +/// What a single NIC pins: the fields a deployment may name, a VM's manifest +/// stores, and `GetInfo` reports back. +/// +/// Separate from [`Networking`] because the node's `[cvm.networking]` is not a +/// NIC -- it is a NIC *plus* the backend settings only the node may set. When +/// the two were one type, resolution copied the whole node value into every +/// VM, so a bridge NIC's manifest entry carried whatever macvtap parent the +/// node happened to have configured, and every consumer that asked "what does +/// this VM pin?" had to know which fields to ignore. Several did not. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct NicNetworking { pub mode: NetworkingMode, // ── Bridge fields ────────────────────────────────────────────── @@ -884,6 +994,44 @@ pub struct Networking { /// Parent host interface for macvtap (e.g., "eth0"). #[serde(default)] pub parent: String, + + // ── Data plane tuning ────────────────────────────────────────── + /// Move packet processing from the QEMU main loop into the host kernel's + /// vhost-net data plane. `None` inherits the node default. Ignored by the + /// user-mode backend, which has no vhost support, and by custom mode, + /// which owns its whole netdev string. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vhost: Option, + /// virtio-net queue pairs. `None` scales with the VM's vCPU count. Only a + /// deployment sets this; there is no node-wide value, because the useful + /// number depends on the VM rather than the host. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queues: Option, + + // ── Ownership markers ────────────────────────────────────────── + /// Take `mode` from node configuration at every launch instead of from + /// this entry. + /// + /// A deployment that only tunes the data plane never named a backend, so + /// the node still owns which one this NIC uses. `mode` is not an `Option` + /// -- every consumer matches on it -- so the entry carries the node's + /// current mode and this flag says not to trust it across a node + /// configuration change. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub inherit_mode: bool, +} + +/// `[cvm.networking]`, and the resolved value a launch hands to QEMU: one NIC +/// plus the backend settings that belong to the node rather than to any VM. +/// +/// Resolution produces this same type because a launch needs both halves; what +/// it must never do is hand the node half back to a VM to store. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct Networking { + #[serde(flatten)] + pub nic: NicNetworking, + + // ── Macvtap fields ──────────────────────────────────────────── /// macvtap forwarding mode. Empty selects "private". #[serde(default)] pub macvtap_mode: String, @@ -908,11 +1056,74 @@ pub struct Networking { // ── Custom fields ────────────────────────────────────────────── #[serde(default)] pub netdev: String, + // ── Runtime markers ──────────────────────────────────────────── } impl Networking { pub fn is_bridge(&self) -> bool { - self.mode == NetworkingMode::Bridge + self.nic.mode == NetworkingMode::Bridge + } + + /// Whether the vhost-net data plane applies to this interface. + /// + /// Defaults to disabled: an upgraded node keeps the exact device shape its + /// VMs booted with (one queue pair, userspace virtio) until the operator + /// opts in, because `vhost = true` requires `/dev/vhost-net` to be + /// accessible to the account QEMU runs under — a precondition the VMM + /// cannot verify on the operator's behalf. + pub fn vhost_enabled(&self) -> bool { + self.nic.vhost.unwrap_or(false) && self.supports_vhost() + } + + /// Whether the backend selected by `mode` can carry a vhost-net data plane + /// at all. Custom mode is excluded because the operator supplies the whole + /// netdev string, including any vhost options. + pub fn supports_vhost(&self) -> bool { + Self::mode_supports_vhost(self.nic.mode) + } + + /// The same question about a mode on its own, for a caller deciding + /// whether a request it has not built an entry for yet can be honoured. + pub fn mode_supports_vhost(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::Bridge | NetworkingMode::Macvtap) + } + + /// Whether the backend selected by `mode` can carry more than one queue + /// pair. + /// + /// Custom mode is excluded for the same reason as vhost: the operator + /// supplies the whole netdev string and the VMM cannot edit it, so a + /// multiqueue device line would have nothing to pair with. The RPC refuses + /// such a request, but a node that switches its default to custom must not + /// be able to produce one behind the RPC's back. + pub fn supports_multiqueue(&self) -> bool { + Self::mode_supports_multiqueue(self.nic.mode) + } + + /// The same question about a mode on its own, for a caller deciding + /// whether a request it has not built an entry for yet can be honoured. + pub fn mode_supports_multiqueue(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::Bridge | NetworkingMode::Macvtap) + } + + /// Effective virtio-net queue pair count of a resolved NIC, never below + /// one. Resolution makes the vCPU-scaled default concrete, so an entry that + /// still carries none is read conservatively as single-queue. + pub fn queue_pairs(&self) -> u32 { + if !self.supports_multiqueue() { + return 1; + } + self.nic.queues.unwrap_or(1).max(1) + } + + /// Queue pairs a VM with this many vCPUs gets when it asks for none. + /// + /// The guest driver uses at most one queue pair per vCPU, so the default + /// follows the vCPU count up to a fixed cap. A node that lowers + /// `max_net_queues` below that cap means it, so the default follows it + /// down; raising it above the cap only widens what a caller may request. + pub fn default_queue_pairs(vcpu: u32, max_net_queues: u32) -> u32 { + vcpu.clamp(1, DEFAULT_QUEUE_SCALING_CAP.min(max_net_queues).max(1)) } /// Parse the mac_prefix into bytes. Returns 0-3 bytes. @@ -1198,6 +1409,27 @@ mod tests { .expect("default VMM config should parse") } + /// The ownership marker is additive on disk: manifests and runtime network + /// snapshots written before it existed still load, and an entry that does + /// not carry it serializes exactly as it used to. + #[test] + fn ownership_markers_are_omitted_when_unset_and_default_when_absent() { + let mut networking: Networking = + serde_json::from_str(r#"{"mode":"bridge","bridge":"br0"}"#).unwrap(); + assert!(!networking.nic.inherit_mode); + + let json = serde_json::to_string(&networking).unwrap(); + assert!(!json.contains("inherit_mode"), "{json}"); + + networking.nic.inherit_mode = true; + let json = serde_json::to_string(&networking).unwrap(); + assert!(json.contains(r#""inherit_mode":true"#), "{json}"); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + networking + ); + } + #[test] fn config_validation_accepts_defaults() { let config = default_config(); @@ -1234,6 +1466,42 @@ mod tests { .to_string() .contains("range start")); + // An empty filter tells netd to create an unfiltered TAP, so libvirt + // mode must never carry one. + let mut config = default_config(); + config.cvm.network_filter.mode = NetworkFilterMode::Libvirt; + config.cvm.network_filter.filter = String::new(); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("network_filter.filter")); + + let mut config = default_config(); + config.cvm.max_net_queues = 0; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("max_net_queues")); + + let mut config = default_config(); + config.cvm.max_net_queues = MAX_NET_QUEUES + 1; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("max_net_queues")); + + // The node-wide value is gone; say so rather than ignoring it. + let mut config = default_config(); + config.cvm.networking.nic.queues = Some(4); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("cvm.networking.queues is not a node setting")); + let mut config = default_config(); config.cvm.networking.mac_prefix = "02:not-hex".into(); assert!(config @@ -1270,8 +1538,8 @@ mod tests { .contains("supervisor.sock")); let mut config = default_config(); - config.cvm.networking.mode = NetworkingMode::Bridge; - config.cvm.networking.bridge.clear(); + config.cvm.networking.nic.mode = NetworkingMode::Bridge; + config.cvm.networking.nic.bridge.clear(); assert!(config .validate() .unwrap_err() @@ -1328,3 +1596,68 @@ mod tests { assert_eq!(CvmPlatform::resolve_from_cpuinfo(cpuinfo), CvmPlatform::Tdx); } } + +#[cfg(test)] +mod networking_shape_tests { + use super::{Config, Networking, NetworkingMode, NicNetworking, DEFAULT_CONFIG}; + + /// Splitting the type must not split the wire format. `[cvm.networking]` + /// is flattened, so a node config, a stored manifest and a runtime-networks + /// snapshot written by an earlier build all still parse, and what this + /// build writes is byte-for-byte what the old one did. + #[test] + fn the_split_types_keep_one_flat_serialized_shape() { + let legacy = serde_json::json!({ + "mode": "bridge", + "bridge": "br0", + "parent": "eth0", + "macvtap_mode": "private", + "device": "/dev/tap7", + "mac_prefix": "02:aa:bb", + "net": "10.0.2.0/24", + "dhcp_start": "10.0.2.15", + "restrict": true, + "netdev": "", + "vhost": false, + "queues": 4, + "inherit_mode": true, + }); + + // A resolved value keeps every field, at the same names as before. + let resolved: Networking = serde_json::from_value(legacy.clone()).unwrap(); + assert_eq!(resolved.nic.mode, NetworkingMode::Bridge); + assert_eq!(resolved.nic.bridge, "br0"); + assert_eq!(resolved.nic.queues, Some(4)); + assert!(resolved.nic.inherit_mode); + assert_eq!(resolved.macvtap_mode, "private"); + assert_eq!(resolved.net, "10.0.2.0/24"); + assert!(resolved.restrict); + let round_tripped = serde_json::to_value(&resolved).unwrap(); + assert_eq!(round_tripped["mode"], "bridge"); + assert_eq!(round_tripped["macvtap_mode"], "private"); + assert_eq!(round_tripped["queues"], 4); + + // A manifest entry written by a build that stored the whole thing + // still loads; the node's half is simply dropped on the way in. + let pinned: NicNetworking = serde_json::from_value(legacy).unwrap(); + assert_eq!(pinned.bridge, "br0"); + assert_eq!(pinned.parent, "eth0"); + assert_eq!(pinned.vhost, Some(false)); + let stored = serde_json::to_value(&pinned).unwrap(); + assert_eq!(stored.as_object().unwrap().len(), 6); + assert!(stored.get("macvtap_mode").is_none()); + assert!(stored.get("net").is_none()); + } + + /// The TOML section still deserializes through the flatten. + #[test] + fn the_node_section_still_parses_from_toml() { + use rocket::figment::providers::Format as _; + let config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + assert_eq!(config.cvm.networking.nic.mode, NetworkingMode::User); + } +} diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 55099e81b..d35864dc4 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -77,6 +77,48 @@ struct NetdArgs { /// Override the Unix socket configured in [netd]. #[arg(long)] socket: Option, + /// Inspect a running netd instead of starting one. + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum NetdCommand { + /// List every host interface netd holds. + /// + /// Answers the question a leak is made of -- whose is this interface -- + /// which deriving a name from an identity cannot. + List { + /// Only this VMM instance's interfaces. Defaults to every one netd + /// owns, including those it cannot attribute. + #[arg(long)] + instance: Option, + }, + /// Delete one interface by name. + /// + /// For what nothing else can name: an interface built before netd recorded + /// ownership, or by another netd, whose VM is gone. `netd list` shows these + /// with no instance and no VM, so nothing can derive the sweep that would + /// take them; an operator who can tell what they are says so here. + RemoveInterface { + /// The interface name, as `netd list` prints it. + name: String, + }, + /// Delete every interface netd holds for one VM. + /// + /// For a VM whose VMM will never ask again -- one whose directory was + /// deleted by hand, or whose instance is gone. A VMM sweeps its own VMs on + /// every stop and every removal, and keeps a removal pending until the + /// sweep lands; this is for when no VMM will ever run that sweep. + RemoveVm { + /// The `cvm.instance_id` of the VMM that created them. `netd list` + /// shows it. + #[arg(long)] + instance: String, + /// The VM's ID. + #[arg(long)] + vm: String, + }, } #[derive(ClapArgs)] @@ -192,6 +234,64 @@ async fn log_rotation_task(app: App) { } } +/// Client-side netd subcommands. Talks to the socket like the VMM does, so it +/// needs whatever the socket's permissions ask for and not root. +async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result<()> { + match command { + NetdCommand::List { instance } => { + let interfaces = netd::list(&config.socket, instance.as_deref().unwrap_or_default()) + .await + .context("failed to list netd interfaces")?; + println!( + "{:<16} {:<8} {:<24} {:<38} {:>3}", + "INTERFACE", "KIND", "INSTANCE", "VM", "NIC" + ); + let mut unattributed = 0; + for record in &interfaces { + if record.instance_id.is_none() { + unattributed += 1; + } + println!( + "{:<16} {:<8} {:<24} {:<38} {:>3}", + record.tap, + record.kind, + record.instance_id.as_deref().unwrap_or("-"), + record.vm_id.as_deref().unwrap_or("-"), + record + .nic_index + .map_or_else(|| "-".to_string(), |index| index.to_string()), + ); + } + println!(); + println!("{} interface(s)", interfaces.len()); + if unattributed > 0 { + // Not a fault to fix by hand: an interface built before netd + // recorded ownership, or by another netd, carries no record and + // gets one the next time its VM launches. + println!( + "{unattributed} carry no ownership record; `netd remove-interface` takes \ + one by name" + ); + } + Ok(()) + } + NetdCommand::RemoveInterface { name } => { + netd::remove_interface_named(&config.socket, name) + .await + .with_context(|| format!("failed to remove {name}"))?; + println!("removed {name}"); + Ok(()) + } + NetdCommand::RemoveVm { instance, vm } => { + let removed = netd::remove_all(&config.socket, instance, vm) + .await + .context("failed to remove the VM's interfaces")?; + println!("removed {removed} interface(s) for {vm}"); + Ok(()) + } + } +} + #[rocket::main] async fn main() -> Result<()> { { @@ -219,6 +319,25 @@ async fn main() -> Result<()> { if let Some(socket) = netd_args.socket.as_deref() { netd_config.socket = socket.into(); } + if netd_config.network_filter.is_none() { + // netd and the VMM normally share one vmm.toml, so the node has + // already stated whether its bridge traffic is filtered and with + // what. Reading it here keeps the two from drifting apart, which is + // what a second setting to keep in sync would invite. + // + // A malformed section is an error rather than a default: this is + // the daemon's security policy, and `[cvm.network_filter] mode = + // "Libvirt"` -- which the VMM itself refuses to start on -- must not + // quietly resolve to "filter nothing" here. + netd_config.network_filter = Some( + figment + .extract_inner("cvm.network_filter") + .context("failed to load [cvm.network_filter] for netd")?, + ); + } + if let Some(command) = &netd_args.command { + return run_netd_command(&netd_config, command).await; + } return netd::serve(netd_config).await; } @@ -230,6 +349,7 @@ async fn main() -> Result<()> { // Preserve the existing startup validation. The broader static checks are // opt-in through `check-config` until they have seen wider deployment use. + netd::validate_instance_id(&config.cvm.instance_id)?; config .host_api .validate() diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 931a373ae..ea4166dcc 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -25,10 +25,11 @@ use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; use crate::app::{ - needs_swtpm, resolve_networking, validate_resolved_network, validate_resolved_networks, App, - AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, + mode_carries_ingress, needs_swtpm, resolve_networking, validate_resolved_network, + validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, + VmWorkDir, }; -use crate::config::{CvmConfig, Networking, NetworkingMode}; +use crate::config::{CvmConfig, Networking, NetworkingMode, NicNetworking}; fn hex_sha256(data: &str) -> String { use sha2::Digest; @@ -168,6 +169,62 @@ fn port_mappings_conflict(left: &PortMapping, right: &PortMapping) -> bool { || right.address.is_unspecified()) } +/// The backend each of a VM's NICs resolves to, with an empty list standing for +/// the node default's single NIC. +fn resolved_nic_modes( + networks: &[NicNetworking], + cvm_config: &CvmConfig, + vcpu: u32, +) -> Vec { + let node_default = [cvm_config.networking.nic.clone()]; + let requested = if networks.is_empty() { + &node_default[..] + } else { + networks + }; + requested + .iter() + .map(|networking| resolve_networking(networking, cvm_config, vcpu).nic.mode) + .collect() +} + +/// Rejects a mapping pinned to a NIC that cannot carry it. +/// +/// Both ways of getting that wrong are refused here, at deployment, where the +/// caller is present to be told: a NIC the VM does not have, and one whose +/// backend has no mechanism to publish a host port. Macvtap bypasses the host +/// bridge and a custom netdev is a string the VMM does not interpret, so a +/// mapping pinned to either used to resolve to that NIC and then fall out of +/// every branch that could act on it -- no `hostfwd=`, no netd request, and no +/// warning either. +fn validate_port_mapping_nics(mappings: &[PortMapping], modes: &[NetworkingMode]) -> Result<()> { + let nic_count = modes.len(); + for mapping in mappings { + let Some(index) = mapping.nic_index else { + continue; + }; + let Some(mode) = modes.get(index) else { + bail!( + "port mapping {} {}:{} names NIC {index}, but this VM has {nic_count}", + mapping.protocol.as_str(), + mapping.address, + mapping.from + ); + }; + if !mode_carries_ingress(*mode) { + bail!( + "port mapping {} {}:{} names NIC {index}, which is {} and cannot publish a host \ + port; use a user-mode or bridge NIC", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mode.as_str(), + ); + } + } + Ok(()) +} + fn validate_unique_port_mappings(mappings: &[PortMapping]) -> Result<()> { for (index, mapping) in mappings.iter().enumerate() { if mappings[..index] @@ -216,10 +273,16 @@ pub fn create_manifest_from_vm_config( protocol, from, to, + nic_index: p.nic_index.map(|index| index as usize), }) }) .collect::>>()?; validate_unique_port_mappings(&port_map)?; + let networks = networks_from_vm_config(&request, cvm_config)?; + validate_port_mapping_nics( + &port_map, + &resolved_nic_modes(&networks, cvm_config, request.vcpu), + )?; let app_id = match &request.app_id { Some(id) => id.strip_prefix("0x").unwrap_or(id).to_lowercase(), @@ -268,7 +331,7 @@ pub fn create_manifest_from_vm_config( no_tee: request.no_tee || simulated_tee.is_some(), simulated_tee, swtpm, - networks: networks_from_vm_config(&request, cvm_config)?, + networks, volumes, }) } @@ -352,18 +415,37 @@ fn resolve_volume_source(base: &Path, source: &str) -> Result { fn networking_from_proto( proto: &rpc::NetworkingConfig, cvm_config: &CvmConfig, -) -> Result> { +) -> Result> { let bridge = proto.bridge_name.trim().to_string(); - let mode = match proto.mode.as_str() { - "bridge" => NetworkingMode::Bridge, - "user" => NetworkingMode::User, - "macvtap" => NetworkingMode::Macvtap, - "" if bridge.is_empty() => return Ok(None), - "" => bail!("networking mode is required when bridge is set"), + let parent = proto.parent.trim().to_string(); + // Checked before anything reads `queues`, because the "no override at all" + // arm below returns early: a request of `{queues: 0}` and nothing else + // would otherwise be answered with the vCPU-scaled default -- up to + // sixteen queue pairs -- for a caller who asked for none. + if proto.queues == Some(0) { + bail!("networking queues must be at least 1, or unset to follow the vCPU count"); + } + let tuned = proto.vhost.is_some() || proto.queues.is_some(); + // Naming a bridge or a macvtap parent is naming a backend, and a backend + // needs a mode to go with it. Without one the entry would freeze whichever + // mode the node happened to have, then report a mode-less override still + // carrying a field only one mode accepts -- something nothing can send + // back once the node moves on. + let names_backend = !bridge.is_empty() || !parent.is_empty(); + // A request that only tunes the data plane keeps the node's backend. Node + // policy governs which backend a caller may *choose*, so inheriting one + // must not be denied by it. + let (mode, chosen) = match proto.mode.as_str() { + "bridge" => (NetworkingMode::Bridge, true), + "user" => (NetworkingMode::User, true), + "macvtap" => (NetworkingMode::Macvtap, true), + "" if !names_backend && !tuned => return Ok(None), + "" if !names_backend => (cvm_config.networking.nic.mode, false), + "" => bail!("networking mode is required when a bridge or macvtap parent is set"), "custom" => bail!("custom networking mode is manifest-only"), other => bail!("unsupported networking mode '{other}'"), }; - if !cvm_config.allowed_network_modes.contains(&mode) { + if chosen && !cvm_config.allowed_network_modes.contains(&mode) { bail!( "networking mode '{}' is not allowed by node policy", proto.mode @@ -372,45 +454,161 @@ fn networking_from_proto( if mode != NetworkingMode::Bridge && !bridge.is_empty() { bail!("bridge_name is only valid for bridge networking mode"); } - if mode != NetworkingMode::Macvtap && !proto.parent.trim().is_empty() { + if mode != NetworkingMode::Macvtap && !parent.is_empty() { bail!("parent is only valid for macvtap networking mode"); } - if !proto.macvtap_mode.trim().is_empty() { + // `GetInfo` reports the resolved node defaults, and both vmm-cli and the + // web UI read that, change one field, and send the rest back. Naming a + // value the node would have supplied anyway therefore has to be accepted: + // leaving the field empty already yields exactly it, so echoing it grants + // nothing that policy was withholding. + // Node values are compared trimmed, the way the request's are: node + // configuration validation tolerates surrounding whitespace, and a value + // resolution would supply must not become unsendable over a space. + let node = &cvm_config.networking; + let macvtap_mode = proto.macvtap_mode.trim(); + if !macvtap_mode.is_empty() && macvtap_mode != node.macvtap_mode.trim() { bail!("macvtap_mode is node-controlled and cannot be set by deployment RPCs"); } - if !bridge.is_empty() && !cvm_config.allowed_bridges.contains(&bridge) { + if !bridge.is_empty() + && bridge != node.nic.bridge.trim() + && !cvm_config.allowed_bridges.contains(&bridge) + { bail!("bridge_name '{bridge}' is not allowed by node policy"); } - let parent = proto.parent.trim().to_string(); - if !parent.is_empty() && !cvm_config.allowed_macvtap_parents.contains(&parent) { + if !parent.is_empty() + && parent != node.nic.parent.trim() + && !cvm_config.allowed_macvtap_parents.contains(&parent) + { bail!("macvtap parent '{parent}' is not allowed by node policy"); } - Ok(Some(Networking { + // Same rule as the queue count below: a backend the caller chose and that + // has no vhost data plane is a request they can fix, so say so. An + // inherited one is not, and reads as off until the node moves. + if chosen && proto.vhost == Some(true) && !Networking::mode_supports_vhost(mode) { + bail!("{} networking has no vhost data plane", proto.mode); + } + // Queue pairs cost a host vhost thread and a pair of MSI-X vectors each, so + // the node caps what a deployment may ask for. + // + let queues = proto.queues; + if let Some(queues) = queues { + if queues > cvm_config.max_net_queues { + bail!( + "networking queues must not exceed {} on this node", + cvm_config.max_net_queues + ); + } + // Only a backend the caller *chose* is theirs to be wrong about. One + // they inherited can change under them -- that is the point of + // inheriting -- and refusing the request afterwards would strand the + // VM: GetInfo would keep reporting a queue count that nothing is + // allowed to send back. `queue_pairs()` already reads as one on both + // of these, so the request simply lies dormant until the node moves to + // a backend that can honour it. + if chosen && queues > 1 && !Networking::mode_supports_multiqueue(mode) { + bail!("{} networking does not support multiple queues", proto.mode); + } + } + // Every field the node owns -- the macvtap forwarding mode, the MAC prefix, + // the user-mode network parameters, a custom netdev string -- is absent by + // construction now rather than by remembering to write `String::new()` for + // each of them. + Ok(Some(NicNetworking { mode, + // The caller named no backend, so the node keeps owning which one this + // NIC uses. `mode` above is only the node's current choice. + inherit_mode: !chosen, bridge, parent, - // The forwarding mode is always inherited from node configuration. - macvtap_mode: String::new(), - device: String::new(), - mac_prefix: String::new(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, - netdev: String::new(), + vhost: proto.vhost, + queues, })) } +/// Networking modes a client should offer. +/// +/// A mode the host could serve but node policy forbids is not one of them: +/// offering it puts a choice in the deploy dialog whose only outcome is "not +/// allowed by node policy", with nothing for the operator to do about it. A VM +/// can still be given a data plane override without naming any mode, which is +/// how a node whose own backend is not caller-selectable stays tunable. +fn advertised_modes(cvm_config: &CvmConfig, host_can_bridge: bool) -> Vec { + [ + (NetworkingMode::User, "user", true), + (NetworkingMode::Bridge, "bridge", host_can_bridge), + (NetworkingMode::Macvtap, "macvtap", true), + ] + .into_iter() + .filter(|(mode, _, host_supports)| { + *host_supports && cvm_config.allowed_network_modes.contains(mode) + }) + .map(|(_, name, _)| name.to_string()) + .collect() +} + +/// The node's policy, widened by what this VM's own NICs already pin. +/// +/// Deployment allowlists govern what a caller may *newly* select. A value one +/// of this VM's interfaces is already running on was selected when it was +/// still allowed, and it is reported back on every `GetInfo`; refusing it +/// would strand the VM rather than withhold anything. +fn held_networking_config(cvm_config: &CvmConfig, held: &[NicNetworking]) -> CvmConfig { + let mut widened = cvm_config.clone(); + for networking in held { + // Only a field the entry's own mode owns. Resolution starts from the + // node's whole `[cvm.networking]` value, so a bridge entry also carries + // whatever macvtap parent the node happened to have configured, and a + // macvtap entry carries its bridge -- baggage the VM never used. Reading + // those as "already running on it" would widen policy from a value + // nothing ever attached to, and let a VM move to a backend the node + // forbids. + if networking.mode == NetworkingMode::Bridge && !networking.bridge.is_empty() { + widened.allowed_bridges.push(networking.bridge.clone()); + } + if networking.mode == NetworkingMode::Macvtap && !networking.parent.is_empty() { + widened + .allowed_macvtap_parents + .push(networking.parent.clone()); + } + // Same rule for the queue cap. Lowering `max_net_queues` bounds what a + // deployment may newly ask for; it does not retroactively re-tune VMs + // that are already pinned above it, and those keep reporting their + // count on every `GetInfo`. Without this, lowering the cap makes every + // networking update on such a VM fail on a field the operator never + // typed -- including the `--net-queues auto` that would unpin it. + if let Some(queues) = networking.queues { + widened.max_net_queues = widened.max_net_queues.max(queues); + } + } + widened +} + +/// A NIC that overrides nothing: whatever the node's `[cvm.networking]` says, +/// now and after the operator changes it. +fn node_default_networking(cvm_config: &CvmConfig) -> NicNetworking { + NicNetworking { + mode: cvm_config.networking.nic.mode, + inherit_mode: true, + ..NicNetworking::default() + } +} + fn network_from_required_proto( proto: &rpc::NetworkingConfig, cvm_config: &CvmConfig, -) -> Result { - networking_from_proto(proto, cvm_config)?.context("networking mode is required") +) -> Result { + // An entry in a list that overrides nothing is not a missing mode: it is a + // NIC that follows the node entirely. Only the singular `networking` field + // can mean "no override at all", because there the absence is the message. + Ok(networking_from_proto(proto, cvm_config)? + .unwrap_or_else(|| node_default_networking(cvm_config))) } fn networks_from_proto( networks: &[rpc::NetworkingConfig], cvm_config: &CvmConfig, -) -> Result> { +) -> Result> { networks .iter() .map(|network| network_from_required_proto(network, cvm_config)) @@ -422,15 +620,68 @@ fn validate_default_network(cvm_config: &CvmConfig) -> Result<()> { } fn resolve_requested_networks( - networks: &[Networking], + requests: &[NicNetworking], cvm_config: &CvmConfig, -) -> Result> { - let resolved = networks + vcpu: u32, +) -> Result> { + let merged = requests .iter() - .map(|networking| resolve_networking(networking, cvm_config)) + .map(|request| resolve_networking(request, cvm_config, vcpu)) .collect::>(); - validate_resolved_networks(&resolved)?; - Ok(resolved) + // Validate the merged view, because that is what the launch sees, then + // record the narrower view the manifest keeps. + validate_resolved_networks(&merged)?; + Ok(manifest_networks(merged, requests)) +} + +/// What a deployment records against the VM, given the merged view its launch +/// would see. +/// +/// The backend a deployment *chose* is pinned here -- its mode, and the bridge +/// or macvtap parent that names it -- so a VM keeps the segment it was put on +/// for life. Everything else stays owned by the node and is re-read at every +/// launch: the MAC prefix, the user-mode subnet and DHCP start, the macvtap +/// forwarding mode, and the data plane an operator may want to roll back +/// node-wide. A VM's own record cannot carry any of those. +fn manifest_networks(merged: Vec, requests: &[NicNetworking]) -> Vec { + merged + .into_iter() + .zip(requests) + .map(|(entry, request)| { + if request.inherit_mode { + // The caller named no backend, so none of the node's matching + // identity fields are theirs to keep. + return request.clone(); + } + // Taking `entry.nic` rather than the whole resolved value is what + // keeps the node's half out of the VM's record; it used to take the + // whole thing and then clear the one node field anybody noticed. + // + // The two identity fields still share one type, and resolution + // fills both from the node, so scope them here -- once, where the + // record is written -- rather than leaving every later reader to + // remember which one its mode owns. Two already have to. + let mode = entry.nic.mode; + NicNetworking { + bridge: if mode == NetworkingMode::Bridge { + entry.nic.bridge + } else { + String::new() + }, + parent: if mode == NetworkingMode::Macvtap { + entry.nic.parent + } else { + String::new() + }, + // Data plane tuning is not identity: leave what the caller did + // not ask for unset. + vhost: request.vhost, + queues: request.queues, + mode, + inherit_mode: entry.nic.inherit_mode, + } + }) + .collect() } fn has_host_bridge_interface() -> bool { @@ -445,13 +696,13 @@ fn has_host_bridge_interface() -> bool { fn networks_from_vm_config( request: &VmConfiguration, cvm_config: &CvmConfig, -) -> Result> { +) -> Result> { if !request.networks.is_empty() { let networks = networks_from_proto(&request.networks, cvm_config)?; - resolve_requested_networks(&networks, cvm_config) + resolve_requested_networks(&networks, cvm_config, request.vcpu) } else if let Some(networking) = request.networking.as_ref() { match networking_from_proto(networking, cvm_config)? { - Some(networking) => resolve_requested_networks(&[networking], cvm_config), + Some(networking) => resolve_requested_networks(&[networking], cvm_config, request.vcpu), None => Ok(vec![]), } } else { @@ -677,6 +928,21 @@ impl VmmRpc for RpcHandler { async fn update_vm(self, request: UpdateVmRequest) -> Result { info!(vm_id = %request.id, "update_vm RPC called"); + // A VM being removed is not one to reconfigure. Before the lock, + // because removal holds it across the whole teardown and anything that + // only asked afterwards would wait that out in order to be told no. + self.app.refuse_if_removing(&request.id)?; + // Held from here rather than around the parts that touch the host, + // because everything below writes into the workdir -- the compose + // file first, the manifest last -- and `put_manifest` creates the + // directory it writes into. An update that resumed after a removal + // deleted that directory would recreate it holding nothing but a + // manifest: invisible to `list_vms`, unloadable at every start, and + // claiming the VM's netd interfaces against collection forever. + let _launch = self.app.launch_lock(&request.id).await; + // Again under the lock: removal can have claimed the VM while this + // waited for it. + self.app.refuse_if_removing(&request.id)?; let new_id = if !request.compose_file.is_empty() { // check the compose file is valid let _app_compose: AppCompose = @@ -730,6 +996,7 @@ impl VmmRpc for RpcHandler { protocol: p.protocol.parse().context("Invalid protocol")?, from: p.host_port.try_into().context("Invalid host port")?, to: p.vm_port.try_into().context("Invalid vm port")?, + nic_index: p.nic_index.map(|index| index as usize), }) }) .collect::>>()?; @@ -747,25 +1014,45 @@ impl VmmRpc for RpcHandler { validate_default_network(&self.app.config.cvm)?; vec![] } else { - let networks = networks_from_proto(&request.networks, &self.app.config.cvm)?; - resolve_requested_networks(&networks, &self.app.config.cvm)? + // A bridge or parent this VM already holds is not a new grant. + // `GetInfo` keeps reporting it, and read-modify-write keeps + // sending it back, so refusing it once the node changes its own + // default would make the VM's configuration unsendable -- with + // no flag anywhere to clear a field the caller never typed. + let cvm = held_networking_config(&self.app.config.cvm, &manifest.networks); + let networks = networks_from_proto(&request.networks, &cvm)?; + resolve_requested_networks(&networks, &cvm, manifest.vcpu)? }; + // Under the launch lock this whole call holds. Reading "not + // running" outside it and acting on the answer inside is the exact + // race the lock exists to close: a launch can start, prepare its + // interfaces and deploy QEMU in between, and the release would + // then delete the interfaces of a VM that is running -- silently, + // since QEMU stays up and the supervisor still reports it healthy. let is_running = self .app .supervisor .info(&request.id) .await? .is_some_and(|info| info.state.status.is_running()); - if !is_running { - let runtime_networks = vm_work_dir.runtime_networks(); - self.app - .remove_filtered_networks(&request.id, &runtime_networks) - .await - .context("failed to remove previous filtered networking")?; + if !is_running && self.app.release_vm_interfaces(&request.id).await { vm_work_dir.clear_runtime_networks()?; } manifest.networks = networks; } + // Both only when this request moved one of the two halves, and after + // both, since either half can move and the other still has to agree + // with it. A VM deployed before the node could answer for its ports + // must stay editable in every other respect: read-modify-write sends + // the whole configuration back, and refusing a memory change over a + // port mapping nobody touched -- or over a node default that changed + // under it -- would make the VM unmanageable rather than fixed. + if request.update_ports || request.update_networking { + validate_port_mapping_nics( + &manifest.port_map, + &resolved_nic_modes(&manifest.networks, &self.app.config.cvm, manifest.vcpu), + )?; + } let compose_file = fs::read_to_string(vm_work_dir.app_compose_path()) .context("failed to read app compose for swtpm decision")?; manifest.swtpm = needs_swtpm( @@ -859,14 +1146,12 @@ impl VmmRpc for RpcHandler { } async fn get_meta(self) -> Result { - let mut supported_modes = vec!["user".to_string()]; let default_networking = &self.app.config.cvm.networking; let mut bridge_networking = default_networking.clone(); - bridge_networking.mode = NetworkingMode::Bridge; - if validate_resolved_network(&bridge_networking).is_ok() || has_host_bridge_interface() { - supported_modes.push("bridge".to_string()); - } - supported_modes.push("macvtap".to_string()); + bridge_networking.nic.mode = NetworkingMode::Bridge; + let host_can_bridge = + validate_resolved_network(&bridge_networking).is_ok() || has_host_bridge_interface(); + let supported_modes = advertised_modes(&self.app.config.cvm, host_can_bridge); Ok(GetMetaResponse { kms: Some(KmsSettings { url: self @@ -900,13 +1185,15 @@ impl VmmRpc for RpcHandler { }), networking: Some(rpc::NetworkingCapabilities { supported_modes, - default_mode: match default_networking.mode { + default_mode: match default_networking.nic.mode { NetworkingMode::User => "user".to_string(), NetworkingMode::Bridge => "bridge".to_string(), NetworkingMode::Custom => String::new(), NetworkingMode::Macvtap => "macvtap".to_string(), }, - default_bridge: default_networking.bridge.clone(), + default_bridge: default_networking.nic.bridge.clone(), + max_queues: self.app.config.cvm.max_net_queues, + default_vhost: default_networking.vhost_enabled(), }), }) } @@ -1150,6 +1437,67 @@ mod tests { } } + fn pinned(nic_index: Option) -> PortMapping { + PortMapping { + address: "0.0.0.0".parse().unwrap(), + protocol: crate::config::Protocol::Tcp, + from: 443, + to: 443, + nic_index, + } + } + + /// Both ways of naming a NIC that cannot publish a port are refused where + /// the caller is present to be told. A pin to macvtap or to a custom + /// netdev used to resolve to that NIC and then fall out of every branch + /// that could act on it, so the port simply never appeared. + #[test] + fn a_pin_is_checked_against_the_backend_and_not_only_the_count() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.parent = "eth0".into(); + cvm.networking.nic.bridge = "br0".into(); + + let bridge_then_macvtap = vec![ + NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + ..NicNetworking::default() + }, + NicNetworking { + mode: NetworkingMode::Macvtap, + parent: "eth0".into(), + ..NicNetworking::default() + }, + ]; + let modes = resolved_nic_modes(&bridge_then_macvtap, &cvm, 2); + assert_eq!(modes, vec![NetworkingMode::Bridge, NetworkingMode::Macvtap]); + + // A bridge TAP is netd's, and netd does not forward host ports. + let error = validate_port_mapping_nics(&[pinned(Some(0))], &modes).unwrap_err(); + assert!(error.to_string().contains("bridge"), "{error}"); + + let error = validate_port_mapping_nics(&[pinned(Some(1))], &modes).unwrap_err(); + assert!(error.to_string().contains("macvtap"), "{error}"); + assert!( + error.to_string().contains("cannot publish a host port"), + "{error}" + ); + + let error = validate_port_mapping_nics(&[pinned(Some(2))], &modes).unwrap_err(); + assert!(error.to_string().contains("this VM has 2"), "{error}"); + + // An unpinned mapping is never refused here: where it lands is + // resolved at launch, from the topology in force then. + validate_port_mapping_nics(&[pinned(None)], &modes).unwrap(); + + // An empty list is the node default's one NIC, resolved the same way a + // launch would resolve it rather than assumed to be user mode. + cvm.networking.nic.mode = NetworkingMode::Macvtap; + let modes = resolved_nic_modes(&[], &cvm, 2); + assert_eq!(modes, vec![NetworkingMode::Macvtap]); + assert!(validate_port_mapping_nics(&[pinned(Some(0))], &modes).is_err()); + } + #[test] fn create_without_networking_persists_following_default() { let manifest = @@ -1282,7 +1630,9 @@ mod tests { assert_eq!(manifest.networks.len(), 1); assert_eq!(manifest.networks[0].mode, NetworkingMode::User); - assert!(!manifest.networks[0].net.is_empty()); + // The node's user-mode network parameters are not the VM's to store; + // the type it stores cannot carry them at all now. + assert!(manifest.networks[0].bridge.is_empty()); } #[test] @@ -1292,7 +1642,7 @@ mod tests { .allowed_network_modes .push(NetworkingMode::Macvtap); cvm_config.allowed_macvtap_parents.push("eth0".to_string()); - cvm_config.networking.parent = "node-default".to_string(); + cvm_config.networking.nic.parent = "node-default".to_string(); cvm_config.networking.macvtap_mode = "private".to_string(); let networks = networks_from_proto( &[rpc::NetworkingConfig { @@ -1306,11 +1656,792 @@ mod tests { assert_eq!(networks[0].mode, NetworkingMode::Macvtap); assert_eq!(networks[0].parent, "eth0"); - assert!(networks[0].macvtap_mode.is_empty()); - let resolved = resolve_requested_networks(&networks, &cvm_config).unwrap(); - assert_eq!(resolved[0].parent, "eth0"); - assert_eq!(resolved[0].macvtap_mode, "private"); + // The manifest keeps the parent, which is identity. The forwarding mode + // the node owns and supplies at every launch is not a field this type + // has. + let stored = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + assert_eq!(stored[0].parent, "eth0"); + + let at_launch = resolve_networking(&stored[0], &cvm_config, 4); + assert_eq!(at_launch.nic.parent, "eth0"); + assert_eq!(at_launch.macvtap_mode, "private"); + + // Repointing the node's forwarding mode reaches the VM. + cvm_config.networking.macvtap_mode = "bridge".to_string(); + assert_eq!( + resolve_networking(&stored[0], &cvm_config, 4).macvtap_mode, + "bridge" + ); + } + + /// Node shapes a deployment can be reported against, each with the node + /// default fields that mode populates. + fn node_shapes() -> Vec<(&'static str, CvmConfig)> { + let mut bridge = test_cvm_config(); + bridge.networking.nic.mode = NetworkingMode::Bridge; + bridge.networking.nic.bridge = "br-node".into(); + + let mut macvtap = test_cvm_config(); + macvtap.networking.nic.mode = NetworkingMode::Macvtap; + macvtap.networking.nic.parent = "eth-node".into(); + macvtap.networking.macvtap_mode = "private".into(); + macvtap.allowed_network_modes.push(NetworkingMode::Macvtap); + + let user = test_cvm_config(); + vec![ + ("bridge node", bridge), + ("macvtap node", macvtap), + ("user node", user), + ] + } + + /// Every override a caller can express, including the tuning-only shape + /// that names no backend. + fn request_shapes(node: &CvmConfig) -> Vec<(String, rpc::NetworkingConfig)> { + let mode = networking_mode_name_for_test(node.networking.nic.mode); + // The user-mode backend has neither multiqueue nor vhost, and naming it + // and then asking for either is a refusal rather than a round-trip + // failure. Asking to turn vhost off is always legal. + let multiqueue = Networking::mode_supports_multiqueue(node.networking.nic.mode); + let vhost_on = Networking::mode_supports_vhost(node.networking.nic.mode).then_some(true); + // Naming a backend's identity field is only legal alongside a mode, so + // it varies with the named case. The node's own values are used because + // that is what GetInfo reports back. + let identity: &[(&str, &str, &str)] = &[ + ("", "", ""), + ("+ bridge", node.networking.nic.bridge.as_str(), ""), + ("+ parent", "", node.networking.nic.parent.as_str()), + ]; + let mut shapes = vec![]; + for (named, mode) in [("named", mode.to_string()), ("inherited", String::new())] { + for (tuning, vhost, queues) in [ + ("untuned", None, None), + ("vhost off", Some(false), None), + ("queues", None, multiqueue.then_some(2)), + ("both", vhost_on, multiqueue.then_some(2)), + ] { + for (label, bridge, parent) in identity { + // An inherited entry may not name a backend at all, which + // is a refusal covered by its own test. + let inherited = mode.is_empty(); + if inherited && !(bridge.is_empty() && parent.is_empty()) { + continue; + } + // Only the mode that owns a field may carry it. + let bridge_ok = node.networking.nic.mode == NetworkingMode::Bridge; + let parent_ok = node.networking.nic.mode == NetworkingMode::Macvtap; + if (!bridge.is_empty() && !bridge_ok) || (!parent.is_empty() && !parent_ok) { + continue; + } + shapes.push(( + format!("{named} + {tuning} {label}"), + rpc::NetworkingConfig { + mode: mode.clone(), + bridge_name: bridge.to_string(), + parent: parent.to_string(), + vhost, + queues, + ..Default::default() + }, + )); + } + } + } + shapes + } + + /// A backend's identity field without a mode would freeze whichever mode + /// the node had at deploy time, and then be reported alongside an empty + /// mode -- a combination the RPC itself rejects, which would leave the VM's + /// tuning permanently uneditable. + #[test] + fn an_inherited_entry_may_not_name_a_backend() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Macvtap; + cvm.networking.nic.parent = "eth-node".into(); + cvm.allowed_macvtap_parents.push("eth1".into()); + cvm.allowed_network_modes.push(NetworkingMode::Macvtap); + + let err = networking_from_proto( + &rpc::NetworkingConfig { + parent: "eth1".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap_err(); + assert!(err.to_string().contains("networking mode is required")); + + // Naming the mode alongside it is fine. + networking_from_proto( + &rpc::NetworkingConfig { + mode: "macvtap".into(), + parent: "eth1".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("a named backend is an override"); + } + + /// Deliberately restated rather than calling `NetworkingMode::as_str`: the + /// test below checks that what `GetInfo` reports is accepted back, and a + /// helper that shares the production mapping could only ever agree with it. + fn networking_mode_name_for_test(mode: NetworkingMode) -> &'static str { + match mode { + NetworkingMode::Bridge => "bridge", + NetworkingMode::User => "user", + NetworkingMode::Macvtap => "macvtap", + NetworkingMode::Custom => "custom", + } + } + + /// `GetInfo` reports the configuration that `UpdateVm` and `UpgradeApp` + /// take back, and both vmm-cli and the web UI read it, change one field, + /// and resend the rest. So everything reportable has to be acceptable, and + /// accepting it has to land on the same VM. + /// + /// This asserts the property over every mode and tuning combination on + /// purpose. Asserting one shape is how an inherited `parent` on a bridge + /// NIC, and then `macvtap_mode` and `bridge_name`, each reached a release: + /// every one of them was a case the fixed example did not cover. + #[test] + fn everything_get_info_reports_is_accepted_back_unchanged() { + for (node_label, cvm) in node_shapes() { + for (shape_label, request) in request_shapes(&cvm) { + let case = format!("{node_label} / {shape_label}"); + let Some(requested) = networking_from_proto(&request, &cvm) + .unwrap_or_else(|error| panic!("{case}: deployment rejected: {error:#}")) + else { + // No override at all; nothing is recorded, nothing to report. + continue; + }; + // Skip the host-dependent bridge existence check: this is + // about what the RPC reports versus what it accepts. + let merged = resolve_networking(&requested, &cvm, 4); + let stored = manifest_networks(vec![merged.clone()], &[requested]); + + let reported = crate::app::networking_to_proto(&stored[0]); + let accepted = networking_from_proto(&reported, &cvm) + .unwrap_or_else(|error| { + panic!("{case}: GetInfo output was rejected on the way back: {error:#}") + }) + .unwrap_or_else(|| panic!("{case}: the override was lost in the round trip")); + + let restored = + manifest_networks(vec![resolve_networking(&accepted, &cvm, 4)], &[accepted]); + assert_eq!(stored, restored, "{case}: round trip changed the manifest"); + assert_eq!( + resolve_networking(&restored[0], &cvm, 4), + merged, + "{case}: round trip changed what the launch sees" + ); + } + } + } + + /// A VM must not become uneditable because its node moved somewhere its + /// tuning does not apply. The report and the deployment RPC have to agree + /// on every backend the node can be pointed at, not just the one it had + /// when the VM was deployed. + #[test] + fn an_inherited_override_still_round_trips_after_the_node_moves() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-node".into(); + + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(4), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("tuning must produce an override"); + let stored = manifest_networks(vec![resolve_networking(&requested, &cvm, 8)], &[requested]); + + for mode in [ + NetworkingMode::User, + NetworkingMode::Custom, + NetworkingMode::Macvtap, + NetworkingMode::Bridge, + ] { + cvm.networking.nic.mode = mode; + cvm.networking.nic.parent = "eth-node".into(); + cvm.networking.netdev = "tap,id=net0,ifname=custom0".into(); + + let reported = crate::app::networking_to_proto(&stored[0]); + let accepted = networking_from_proto(&reported, &cvm) + .unwrap_or_else(|error| panic!("{mode:?}: report was rejected: {error:#}")) + .unwrap_or_else(|| panic!("{mode:?}: the override was lost")); + let restored = + manifest_networks(vec![resolve_networking(&accepted, &cvm, 8)], &[accepted]); + // An inherited entry still has to hold *some* mode -- every + // consumer matches on one -- and it is rewritten to whatever the + // node has now. Resolution ignores it, so compare what the launch + // sees rather than the field nothing reads. + assert!(restored[0].inherit_mode, "{mode:?}: pinned a backend"); + assert_eq!( + resolve_networking(&restored[0], &cvm, 8), + resolve_networking(&stored[0], &cvm, 8), + "{mode:?}: round trip changed what the launch sees" + ); + assert_eq!(restored[0].queues, Some(4), "{mode:?}: lost the request"); + } + } + + /// The counterpart: a node that changes its mind still reaches VMs that + /// never named a backend, and never reaches ones that did. + #[test] + fn a_node_backend_change_reaches_exactly_the_vms_that_inherited_it() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-node".into(); + + let tuning_only = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("tuning must produce an override"); + let named = networking_from_proto( + &rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("a named backend is an override"); + + let requests = vec![tuning_only, named]; + let merged = requests + .iter() + .map(|request| resolve_networking(request, &cvm, 4)) + .collect::>(); + let stored = manifest_networks(merged, &requests); + + // The operator repoints the node at a different backend. + cvm.networking.nic.mode = NetworkingMode::User; + assert_eq!( + resolve_networking(&stored[0], &cvm, 4).nic.mode, + NetworkingMode::User, + "a VM that never named a backend must follow the node" + ); + assert_eq!( + resolve_networking(&stored[1], &cvm, 4).nic.mode, + NetworkingMode::Bridge, + "a VM that named its backend keeps it for life" + ); + // User mode has no multiqueue backend, so the inherited NIC drops to + // one queue pair while it is there -- but the request is not lost. + assert_eq!(resolve_networking(&stored[0], &cvm, 4).queue_pairs(), 1); + assert_eq!(stored[0].queues, Some(2)); + cvm.networking.nic.mode = NetworkingMode::Bridge; + assert_eq!( + resolve_networking(&stored[0], &cvm, 4).queue_pairs(), + 2, + "tuning must survive an excursion through a backend that ignores it" + ); + } + + #[test] + fn queue_requests_are_bounded_by_node_policy() { + let mut cvm_config = test_cvm_config(); + cvm_config.max_net_queues = 4; + cvm_config.allowed_bridges.push("tenant-br0".to_string()); + let request = |queues: u32| { + [rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "tenant-br0".to_string(), + queues: Some(queues), + ..Default::default() + }] + }; + + let networks = networks_from_proto(&request(4), &cvm_config).unwrap(); + assert_eq!(networks[0].queues, Some(4)); + + let err = networks_from_proto(&request(5), &cvm_config).unwrap_err(); + assert!(err.to_string().contains("must not exceed 4")); + } + + /// A mode the deploy dialog offers has to be one the deployment RPC will + /// take. Offering one node policy forbids puts a choice in front of an + /// operator whose only outcome is "not allowed by node policy". + #[test] + fn advertised_modes_are_ones_the_rpc_would_accept() { + let mut cvm = test_cvm_config(); + cvm.allowed_network_modes = vec![NetworkingMode::User]; + for mode in ["bridge", "macvtap"] { + assert!( + networking_from_proto( + &rpc::NetworkingConfig { + mode: mode.to_string(), + ..Default::default() + }, + &cvm, + ) + .is_err(), + "{mode} should be refused by this policy" + ); + } + assert_eq!(advertised_modes(&cvm, true), vec!["user".to_string()]); + + cvm.allowed_network_modes = vec![NetworkingMode::User, NetworkingMode::Macvtap]; + assert_eq!( + advertised_modes(&cvm, true), + vec!["user".to_string(), "macvtap".to_string()] + ); + + // A mode policy allows but the host cannot serve is still not offered. + cvm.allowed_network_modes.push(NetworkingMode::Bridge); + assert!(!advertised_modes(&cvm, false).contains(&"bridge".to_string())); + assert!(advertised_modes(&cvm, true).contains(&"bridge".to_string())); + } + + /// A VM keeps its bridge for life, so the node dropping that bridge from + /// its own configuration must not make the VM's reported configuration + /// unsendable -- there is no flag anywhere to clear a field the caller + /// never typed. + /// Deployment allowlists govern what a caller may newly select. Widening + /// them from a VM's own holdings is what keeps read-modify-write working + /// across a node change -- but only from a field the entry's own mode + /// owns. Resolution used to copy the node's whole networking value into + /// every VM, so a bridge NIC carried whatever macvtap parent the node had + /// configured, and widening from that let the VM move to a parent policy + /// forbids. The split types make the node's other fields unreachable; the + /// two identity fields still share one type, so the scoping stays explicit. + #[test] + fn holdings_widen_policy_only_for_the_mode_that_owns_them() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.parent = "eth1".into(); + cvm.allowed_network_modes = vec![ + NetworkingMode::Bridge, + NetworkingMode::Macvtap, + NetworkingMode::User, + ]; + assert!(cvm.allowed_macvtap_parents.is_empty()); + + // What a bridge NIC deployed on this node used to end up holding. + let held = [NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + parent: "eth0".into(), + ..NicNetworking::default() + }]; + let widened = held_networking_config(&cvm, &held); + assert!(widened.allowed_bridges.contains(&"br0".to_string())); + assert!(!widened + .allowed_macvtap_parents + .contains(&"eth0".to_string())); + + let request = [rpc::NetworkingConfig { + mode: "macvtap".into(), + parent: "eth0".into(), + ..Default::default() + }]; + let err = networks_from_proto(&request, &widened).unwrap_err(); + assert!( + err.to_string().contains("not allowed by node policy"), + "{err}" + ); + } + + /// Lowering the node's queue cap bounds what a deployment may newly ask + /// for. It does not retune VMs already pinned above it, and those keep + /// reporting their count on every GetInfo -- so without this, lowering the + /// cap makes every networking update on such a VM fail on a field the + /// operator never typed, including the one that would unpin it. + #[test] + fn a_vm_may_restate_a_queue_count_the_node_has_since_capped() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.max_net_queues = 2; + + let held = [NicNetworking { + mode: NetworkingMode::Bridge, + queues: Some(8), + ..NicNetworking::default() + }]; + let request = [rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(8), + vhost: Some(false), + ..Default::default() + }]; + + let err = networks_from_proto(&request, &cvm).unwrap_err(); + assert!(err.to_string().contains("must not exceed 2"), "{err}"); + + let widened = held_networking_config(&cvm, &held); + let networks = networks_from_proto(&request, &widened).unwrap(); + assert_eq!(networks[0].queues, Some(8)); + + // Only up to what it holds, though. + let more = [rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(9), + ..Default::default() + }]; + let err = networks_from_proto(&more, &widened).unwrap_err(); + assert!(err.to_string().contains("must not exceed 8"), "{err}"); + } + + /// `optional uint32` tells an absent field from a typed zero, so reading + /// zero as "unset" would answer a request for no queues with the + /// vCPU-scaled default. Resize says the same about a zero vCPU count. + #[test] + fn an_explicit_zero_queue_count_is_an_error_not_an_absent_field() { + let cvm = test_cvm_config(); + let request = rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(0), + ..Default::default() + }; + let err = networking_from_proto(&request, &cvm).unwrap_err(); + assert!(err.to_string().contains("must be at least 1"), "{err}"); + + // Including when it is the only thing the request says. That arm + // returns "no override at all" before anything reads the count, so a + // zero here used to be answered with up to sixteen queue pairs. + let bare = rpc::NetworkingConfig { + queues: Some(0), + ..Default::default() + }; + let err = networking_from_proto(&bare, &cvm).unwrap_err(); + assert!(err.to_string().contains("must be at least 1"), "{err}"); + } + + /// Resolution fills both identity fields from the node, so a bridge NIC's + /// resolved value carries whatever macvtap parent the node happens to have + /// configured. Storing that made every later reader responsible for knowing + /// which field its mode owns, and the one that widens deployment policy + /// from a VM's holdings got it wrong: repoint the node's parent and a + /// bridge VM could move itself to a parent policy forbids. + #[test] + fn a_vm_records_only_the_identity_field_its_own_mode_owns() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-node".into(); + cvm.networking.nic.parent = "eth-node".into(); + + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "bridge".into(), + ..Default::default() + }], + &cvm, + ) + .unwrap(); + // `manifest_networks` directly: what a VM records is the question, and + // `resolve_requested_networks` would first validate the merged view + // against this host's real interfaces. + let merged = networks + .iter() + .map(|request| resolve_networking(request, &cvm, 4)) + .collect::>(); + let stored = manifest_networks(merged, &networks); + assert_eq!(stored[0].bridge, "br-node"); + assert!(stored[0].parent.is_empty(), "{:?}", stored[0]); + + // And the launch still gets the node's parent, because that half was + // never the VM's to hold in the first place. + let at_launch = resolve_networking(&stored[0], &cvm, 4); + assert_eq!(at_launch.nic.parent, "eth-node"); + } + + #[test] + fn a_vm_may_restate_a_bridge_it_already_holds() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-new".into(); + assert!(cvm.allowed_bridges.is_empty()); + + let held = [NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br-old".into(), + ..NicNetworking::default() + }]; + let request = [rpc::NetworkingConfig { + mode: "bridge".into(), + bridge_name: "br-old".into(), + queues: Some(2), + ..Default::default() + }]; + + // Without the VM's own holdings this is a bridge it may not select. + let err = networks_from_proto(&request, &cvm).unwrap_err(); + assert!(err.to_string().contains("not allowed by node policy")); + + let widened = held_networking_config(&cvm, &held); + let networks = networks_from_proto(&request, &widened).unwrap(); + assert_eq!(networks[0].bridge, "br-old"); + + // And it is still only this VM's own values that are permitted. + let other = [rpc::NetworkingConfig { + mode: "bridge".into(), + bridge_name: "br-someone-else".into(), + ..Default::default() + }]; + assert!(networks_from_proto(&other, &widened).is_err()); + } + + /// vhost follows the same rule as the queue count: refused for a backend + /// the caller chose and that has none, accepted and dormant for one they + /// inherited. Accepting it silently on a chosen backend would leave the + /// deploy dialog reporting `vhost: on` next to a NIC running without it. + #[test] + fn vhost_is_refused_only_for_a_backend_the_caller_chose() { + let cvm_config = test_cvm_config(); + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + vhost: Some(true), + ..Default::default() + }, + &cvm_config, + ) + .unwrap_err(); + assert!(err.to_string().contains("no vhost data plane"), "{err:#}"); + + // Turning it off is a no-op that matches reality, so it is allowed. + networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + vhost: Some(false), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + + // Inherited from a user-mode node: accepted, dormant, and live again + // when the node moves to a backend that has one. + let mut cvm_config = test_cvm_config(); + assert_eq!(cvm_config.networking.nic.mode, NetworkingMode::User); + let requested = networking_from_proto( + &rpc::NetworkingConfig { + vhost: Some(true), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert!(!resolve_networking(&requested, &cvm_config, 4).vhost_enabled()); + + cvm_config.networking.nic.mode = NetworkingMode::Bridge; + cvm_config.networking.nic.bridge = "br-node".into(); + assert!(resolve_networking(&requested, &cvm_config, 4).vhost_enabled()); + } + + /// A queue count is refused for a backend the caller chose and that cannot + /// honour it, because the caller can fix the request. It is accepted for + /// one they inherited, because they cannot: the node picked that backend + /// and may pick another tomorrow, and refusing would leave GetInfo + /// reporting a count nothing is allowed to send back. + #[test] + fn a_queue_count_is_refused_only_for_a_backend_the_caller_chose() { + let cvm_config = test_cvm_config(); + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + queues: Some(4), + ..Default::default() + }, + &cvm_config, + ) + .unwrap_err(); + assert!(err.to_string().contains("does not support multiple queues")); + + // Inherited: accepted, and dormant until the node moves to a backend + // that can honour it. + for mode in [NetworkingMode::Custom, NetworkingMode::User] { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.mode = mode; + cvm_config.networking.netdev = "tap,id=net0,ifname=custom0".into(); + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(4), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert_eq!(requested.queues, Some(4)); + assert_eq!( + resolve_networking(&requested, &cvm_config, 8).queue_pairs(), + 1, + "{mode:?} cannot carry multiqueue, whatever was asked for" + ); + + // And the request is still there when the node moves back. + cvm_config.networking.nic.mode = NetworkingMode::Bridge; + cvm_config.networking.nic.bridge = "br-node".into(); + assert_eq!( + resolve_networking(&requested, &cvm_config, 8).queue_pairs(), + 4 + ); + } + } + + #[test] + fn user_mode_rejects_multiqueue_but_a_single_queue_is_fine() { + let cvm_config = test_cvm_config(); + let request = |queues: u32| { + [rpc::NetworkingConfig { + mode: "user".to_string(), + queues: Some(queues), + ..Default::default() + }] + }; + + networks_from_proto(&request(1), &cvm_config).unwrap(); + let err = networks_from_proto(&request(2), &cvm_config).unwrap_err(); + assert!(err.to_string().contains("does not support multiple queues")); + } + + #[test] + fn tuning_alone_keeps_the_node_backend_without_tripping_mode_policy() { + let mut cvm_config = test_cvm_config(); + // A backend the node uses but does not let callers choose. + cvm_config.networking.nic.mode = NetworkingMode::Macvtap; + cvm_config.networking.nic.parent = "eth0".to_string(); + assert!(!cvm_config + .allowed_network_modes + .contains(&NetworkingMode::Macvtap)); + + let networking = networking_from_proto( + &rpc::NetworkingConfig { + vhost: Some(false), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert_eq!(networking.mode, NetworkingMode::Macvtap); + assert_eq!(networking.vhost, Some(false)); + + // Naming that backend explicitly is still a choice, and still denied. + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "macvtap".to_string(), + vhost: Some(false), + ..Default::default() + }, + &cvm_config, + ) + .unwrap_err(); + assert!(err.to_string().contains("not allowed by node policy")); + + // An untouched request still means "no override at all". + assert!( + networking_from_proto(&rpc::NetworkingConfig::default(), &cvm_config) + .unwrap() + .is_none() + ); + } + + #[test] + fn an_inherited_backend_is_never_pinned_into_the_manifest() { + // Tuning must not become a way to pin a backend the caller was never + // allowed to choose, nor to freeze one the node still owns. + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.mode = NetworkingMode::Macvtap; + cvm_config.networking.nic.parent = "eth0".to_string(); + cvm_config.networking.macvtap_mode = "private".to_string(); + + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(2), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert!(requested.inherit_mode); + + let persisted = resolve_requested_networks(&[requested], &cvm_config, 4).unwrap(); + assert_eq!(persisted[0].queues, Some(2)); + // Nothing the node owns was copied in. The forwarding mode, the MAC + // prefix and the user-mode network parameters are not fields this type + // has any more; the parent is, and a NIC that named no backend does not + // get to keep the node's. + assert!(persisted[0].parent.is_empty()); + assert!(persisted[0].bridge.is_empty()); + + // Repointing the node moves the VM with it. + cvm_config.networking.nic.parent = "eth1".to_string(); + let at_launch = resolve_networking(&persisted[0], &cvm_config, 4); + assert_eq!(at_launch.nic.parent, "eth1"); + assert_eq!(at_launch.queue_pairs(), 2); + } + + #[test] + fn deployment_pins_identity_but_not_data_plane_tuning() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.vhost = Some(true); + cvm_config.networking.nic.queues = Some(2); + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "user".to_string(), + vhost: Some(false), + ..Default::default() + }], + &cvm_config, + ) + .unwrap(); + + let resolved = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + // The backend the caller chose is pinned. + assert_eq!(resolved[0].mode, NetworkingMode::User); + assert!(!resolved[0].inherit_mode); + // The explicit request is kept. + assert_eq!(resolved[0].vhost, Some(false)); + // What the caller never asked for stays unset, so the node still owns + // it: an operator disabling vhost node-wide must reach this VM too. + assert_eq!(resolved[0].queues, None); + } + + #[test] + fn a_node_wide_vhost_rollback_reaches_a_vm_deployed_with_an_override() { + // macvtap keeps this independent of which interfaces the test host has. + let mut cvm_config = test_cvm_config(); + cvm_config + .allowed_network_modes + .push(NetworkingMode::Macvtap); + cvm_config.allowed_macvtap_parents.push("eth0".to_string()); + cvm_config.networking.nic.parent = "eth0".to_string(); + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "macvtap".to_string(), + parent: "eth0".to_string(), + ..Default::default() + }], + &cvm_config, + ) + .unwrap(); + let persisted = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + assert!(persisted[0].vhost.is_none()); + + cvm_config.networking.nic.vhost = Some(false); + let at_launch = resolve_networking(&persisted[0], &cvm_config, 4); + assert!(!at_launch.vhost_enabled()); } #[test] @@ -1399,18 +2530,35 @@ mod tests { } #[test] - fn repeated_networks_rejects_empty_entries() { - let err = networks_from_proto( + /// An entry in a list that overrides nothing describes a NIC that follows + /// the node entirely -- which is a thing an operator can mean, and the + /// only way the web UI can leave a NIC's backend unpinned. Rejecting it + /// would make an inherited NIC uneditable the moment its tuning is cleared. + fn repeated_networks_accepts_an_entry_that_overrides_nothing() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.mode = NetworkingMode::Bridge; + cvm_config.networking.nic.bridge = "br-node".into(); + + let networks = networks_from_proto( &[rpc::NetworkingConfig { mode: String::new(), bridge_name: String::new(), ..Default::default() }], - &test_cvm_config(), + &cvm_config, ) - .unwrap_err(); + .unwrap(); + assert_eq!(networks.len(), 1); + assert!(networks[0].inherit_mode); + assert_eq!(networks[0].queues, None); + assert!(networks[0].bridge.is_empty()); - assert!(err.to_string().contains("networking mode is required")); + // It follows the node, like a VM with no networks at all. + cvm_config.networking.nic.mode = NetworkingMode::User; + assert_eq!( + resolve_networking(&networks[0], &cvm_config, 4).nic.mode, + NetworkingMode::User + ); } #[test] diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index b7a946133..52aeeb4ae 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -5,7 +5,7 @@ //! Small privileged broker for TAP creation and libvirt nwfilter bindings. use std::{ - collections::BTreeMap, + collections::HashSet, fs::{File, OpenOptions, Permissions}, io::Write as _, os::{ @@ -29,11 +29,11 @@ use tokio::{ net::{UnixListener, UnixStream}, time::timeout, }; -use tracing::{info, warn}; +use tracing::{debug, info, warn}; use uuid::Uuid; use wait_timeout::ChildExt; -use crate::config::NetdConfig; +use crate::config::{NetdConfig, NetworkFilterConfig}; const MAX_MESSAGE_SIZE: u64 = 64 * 1024; const CONNECTION_TIMEOUT: Duration = Duration::from_secs(35); @@ -41,7 +41,23 @@ const COMMAND_TIMEOUT: Duration = Duration::from_secs(30); const IP_PATH: &str = "/usr/sbin/ip"; const VIRSH_PATH: &str = "/usr/bin/virsh"; const LOCK_PATH: &str = "/run/lock/dstack-netd.lock"; - +/// Upper bound on TAP queue pairs netd will create. Mirrors the VMM's own cap +/// so a malformed request cannot ask the kernel for an unbounded device. +const MAX_QUEUES: u32 = 64; +/// Highest NIC index an identity may name. Also the width of the space a +/// whole-VM sweep has to enumerate, since it derives names instead of reading a +/// record. +const MAX_NIC_INDEX: usize = 255; +/// The interface names netd may create. Reserved: anything matching it is +/// netd's to delete, and nothing else on the host may take one. +const TAP_PREFIX: &str = "dt"; +/// Hex characters of digest in a TAP name, after [`TAP_PREFIX`]. +const TAP_DIGEST_CHARS: usize = 12; +/// Version tag on the ownership record. Present so a later format can be told +/// from this one rather than mis-parsed as it. +const ALIAS_PREFIX: &str = "dstack1"; +/// What the kernel stores in an interface alias, minus the terminator. +const MAX_IFALIAS: usize = 255; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InterfaceIdentity { pub instance_id: String, @@ -56,9 +72,48 @@ pub struct PrepareBridgeRequest { pub bridge: String, pub mac: String, pub qemu_uid: u32, - pub filter: String, + /// Whether to bind an nwfilter to the TAP. *Which* filter, and with what + /// parameters, is netd's own configuration to decide -- a caller that named + /// them could name one that filters nothing, or pin the binding to the + /// gateway's MAC and IP, and still satisfy a node policy that only asked + /// for "some filter". An unfiltered TAP is what multiqueue bridge + /// networking needs on nodes that do not run libvirt. + pub filtered: bool, + /// virtio-net queue pairs. Zero or one creates a single-queue TAP. QEMU + /// rejects a device whose `IFF_MULTI_QUEUE` state differs from its own + /// `queues=` argument, so this must match the launch exactly. + pub queues: u32, + /// The VM's working directory on the host, for logs and diagnostics. + /// + /// Untrusted and never read for a decision: any process that can reach the + /// socket can assert anything here. It is carried so an operator reading + /// netd's log can get from an opaque TAP name back to the VM that asked for + /// it without going through the VMM. #[serde(default)] - pub parameters: BTreeMap, + pub workdir: String, +} + +/// One host resource netd holds. +/// +/// `instance_id` and `vm_id` are absent when the interface carries no record +/// that checks out: built by a netd too old to write one, by a third-party +/// netd, or by this one in the instant between creating the interface and +/// recording it. Absent is not "nobody's" -- it is "not known to be anybody's", +/// which is a materially different thing to a collection. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InterfaceRecord { + pub tap: String, + /// `"tap"`, `"macvtap"`, or `"binding"` for an nwfilter binding whose + /// interface is already gone. A binding outlives the interface it was + /// bound to, so a collection that only looked at interfaces would leave + /// the one piece of state that survives them. + pub kind: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub instance_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vm_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nic_index: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -70,6 +125,14 @@ pub struct PrepareMacvtapRequest { pub qemu_uid: u32, #[serde(default)] pub mode: String, + /// virtio-net queue pairs. The device is created with matching hardware + /// queues; QEMU then opens the character device once per queue. + #[serde(default)] + pub queues: u32, + /// The VM's working directory on the host. Informational only; see + /// [`PrepareBridgeRequest::workdir`]. + #[serde(default)] + pub workdir: String, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -81,11 +144,49 @@ pub enum Request { #[serde(flatten)] identity: InterfaceIdentity, }, + /// Delete every interface netd holds for one VM. + /// + /// Teardown by identity can only reach the NIC indices its caller still has + /// a record of, and that record is written after the interface exists: a + /// VMM killed in between leaves a TAP nothing on disk points at. A manifest + /// that lost a NIC leaves the same thing behind. Both are found here + /// without a record, because every name netd can produce for a VM is + /// derivable from its identity. + RemoveAll { + instance_id: String, + vm_id: String, + }, + /// Everything netd holds, so that an operator can see the host's + /// interfaces without being told what to look for. + /// + /// Deriving a name answers "where is this VM's interface". It cannot + /// answer "whose is this interface", which is the question a leak is made + /// of: a VM whose directory was deleted, a VMM instance that was + /// decommissioned, an interface built by a netd that has since been + /// upgraded. Enumeration answers it. + List { + /// Only interfaces recorded as this instance's. Empty lists every one + /// netd owns, whatever it is recorded as and whether or not it is. + #[serde(default)] + instance_id: String, + }, + /// Delete one interface by name. + /// + /// For what nothing else can reach: an interface built before netd recorded + /// ownership, or by another netd, whose VM is gone. Nothing can attribute + /// it, so nothing can decide about it -- but an operator looking at + /// `list` can, and this is how they say so. Guarded the same way every + /// other removal is: the name must be one netd could have created, and the + /// device must be one netd creates. + RemoveInterface { + tap: String, + }, /// Verify a deterministic TAP and binding for operations and integration /// diagnostics. The VMM startup path uses Prepare rather than Check. Check { #[serde(flatten)] identity: InterfaceIdentity, + filtered: bool, }, } @@ -96,17 +197,162 @@ struct Response { tap: Option, #[serde(default, skip_serializing_if = "Option::is_none")] device: Option, + /// Queue pairs the interface was actually created with. Absent from a netd + /// that predates multiqueue, which is how the VMM tells the difference + /// between "one queue was requested" and "this netd ignored the request". + #[serde(default, skip_serializing_if = "Option::is_none")] + queues: Option, + /// How many interfaces a whole-VM sweep deleted. + #[serde(default, skip_serializing_if = "Option::is_none")] + removed: Option, + /// For a listing, everything netd holds; for a collection, what it took, + /// or would take on a dry run. Absent, rather than empty, from a netd that + /// cannot enumerate: "I hold nothing" and "I cannot say" are answers a + /// collection must not confuse. + #[serde(default, skip_serializing_if = "Option::is_none")] + interfaces: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, } +/// What one request produced. +/// +/// An enum rather than a struct of options, because the shapes do not overlap: +/// a prepare names an interface, a sweep counts them, a listing enumerates. +/// One flat [`Response`] still carries all of them on the wire, so a netd that +/// grows an operation stays readable to a caller that does not know it. +enum Outcome { + /// One interface, named because the caller hands that name to QEMU. + Interface { + tap: String, + device: Option, + queues: Option, + }, + /// A sweep names no single interface, so it reports how many it deleted. + Swept { + removed: usize, + }, + Listed(Vec), +} + +impl Outcome { + fn tap(tap: String) -> Self { + Self::Interface { + tap, + device: None, + queues: None, + } + } + + fn into_response(self) -> Response { + let mut response = Response { + ok: true, + tap: None, + device: None, + queues: None, + removed: None, + interfaces: None, + error: None, + }; + match self { + Self::Interface { + tap, + device, + queues, + } => { + response.tap = Some(tap); + response.device = device; + response.queues = queues; + } + Self::Swept { removed } => response.removed = Some(removed), + Self::Listed(interfaces) => response.interfaces = Some(interfaces), + } + response + } +} + pub fn tap_name(identity: &InterfaceIdentity) -> String { let input = format!( "{}\0{}\0{}", identity.instance_id, identity.vm_id, identity.nic_index ); let digest = Sha256::digest(input.as_bytes()); - format!("dt{}", hex::encode(&digest[..6])) + format!( + "{TAP_PREFIX}{}", + hex::encode(&digest[..TAP_DIGEST_CHARS / 2]) + ) +} + +/// The ownership record netd writes onto every interface it creates. +/// +/// The record lives on the resource, so it has exactly the resource's +/// lifetime. A file under `/run` would be a second thing to keep in step with +/// the first, and the failure this whole path exists to fix is precisely a +/// record that got out of step: written after the interface, lost with the +/// directory, and unreadable to anything but the process that wrote it. +/// +/// Never trusted as *authority*. Anything that can reach this socket can also +/// name an identity, and the interface name is a digest of that identity -- +/// so a record is believed only when re-deriving the name from it reproduces +/// the name it is written on. Ambiguity (a separator inside an identity), +/// truncation, and forgery all fail that check and land in the same bucket as +/// no record at all, which is the bucket handled conservatively. +pub fn interface_alias(identity: &InterfaceIdentity) -> String { + format!( + "{ALIAS_PREFIX}:{}:{}:{}", + identity.nic_index, identity.instance_id, identity.vm_id + ) +} + +/// The identity an interface claims, if the claim checks out. +/// +/// `nic_index` first, so the two free-form fields are the last two and a +/// `vm_id` containing the separator still parses. An `instance_id` containing +/// one does not, and is refused at prepare rather than mis-parsed here. +pub fn owner_of(tap: &str, alias: &str) -> Option { + // `trim_end_matches`, not `trim`: sysfs adds a newline, and a `vm_id` + // whose own trailing whitespace were trimmed off here would re-derive a + // name that is not the one it is on, making the interface permanently + // unattributable -- never collected, only removable by hand. + let rest = alias + .trim_end_matches(['\n', '\r']) + .strip_prefix(ALIAS_PREFIX)? + .strip_prefix(':')?; + let (nic_index, rest) = rest.split_once(':')?; + let (instance_id, vm_id) = rest.split_once(':')?; + let identity = InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + nic_index: nic_index.parse().ok()?, + }; + // The name is the proof. A record that does not reproduce it describes + // some other interface, or nothing. + (tap_name(&identity) == tap).then_some(identity) +} + +/// Whether this name is one netd can have created. See [`TAP_PREFIX`]. +pub fn is_managed_name(interface: &str) -> bool { + let Some(digest) = interface.strip_prefix(TAP_PREFIX) else { + return false; + }; + digest.len() == TAP_DIGEST_CHARS + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Rejects an instance ID no interface could be recorded as belonging to. +/// +/// At startup rather than at the first launch. The VMM derives one that is +/// always valid; an operator who configured their own learns here rather than +/// from the first VM that fails to get a NIC. +pub fn validate_instance_id(instance_id: &str) -> Result<()> { + validate_identity(&InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: "0".repeat(64), + nic_index: MAX_NIC_INDEX, + }) + .context("invalid cvm.instance_id") } pub fn instance_id(configured: &str, run_path: &Path) -> String { @@ -119,19 +365,116 @@ pub fn instance_id(configured: &str, run_path: &Path) -> String { pub struct PreparedInterface { pub device: Option, + pub queues: Option, +} + +/// Marker carried in the error chain when the VMM could not reach netd at all. +/// +/// "netd refused this" and "netd is not there" call for different advice, and +/// the caller cannot tell them apart from the message alone -- a callback probe +/// afterwards would answer about a different moment. +#[derive(Debug)] +pub struct Unreachable; + +impl std::fmt::Display for Unreachable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("netd is not reachable") + } +} + +impl std::error::Error for Unreachable {} + +/// Whether this error means netd was never reached. +/// +/// `downcast_ref` rather than a walk over `chain()`: a marker attached with +/// `context` is not a link in the source chain, it is the context *of* a link, +/// and `chain()` yields the wrapper rather than the marker inside it. Asking +/// the chain therefore always answered no -- which made every "an unreachable +/// netd is not a failure" branch in this crate unreachable itself. +pub fn is_unreachable(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() } pub async fn request(socket: &Path, request: &Request) -> Result { + let response = exchange(socket, request).await?; + if response.tap.as_deref().unwrap_or_default().is_empty() { + bail!("netd response omitted TAP name"); + } + Ok(PreparedInterface { + device: response.device, + queues: response.queues, + }) +} + +/// Deletes every interface netd holds for one VM, returning how many there +/// were. See [`Request::RemoveAll`]. +/// +/// A netd that answers without a count did not sweep. Reading that as zero +/// would report a netd that cannot do this as a VM that had nothing to remove, +/// which is a netd that cannot do this reported as a VM that had nothing to +/// remove -- so it is an error here. +pub async fn remove_all(socket: &Path, instance_id: &str, vm_id: &str) -> Result { + let request = Request::RemoveAll { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + }; + exchange(socket, &request) + .await? + .removed + .context("netd answered a sweep without saying what it removed") +} + +/// Deletes one interface by name. See [`Request::RemoveInterface`]. +pub async fn remove_interface_named(socket: &Path, tap: &str) -> Result<()> { + let request = Request::RemoveInterface { + tap: tap.to_string(), + }; + exchange(socket, &request).await.map(|_| ()) +} + +/// Everything netd holds, optionally narrowed to one VMM instance. See +/// [`Request::List`]. +pub async fn list(socket: &Path, instance_id: &str) -> Result> { + let request = Request::List { + instance_id: instance_id.to_string(), + }; + exchange(socket, &request) + .await? + .interfaces + .context("netd answered a listing without one") +} + +async fn exchange(socket: &Path, request: &Request) -> Result { let operation = match request { Request::PrepareBridge(_) => "prepare_bridge", Request::PrepareMacvtap(_) => "prepare_macvtap", Request::Remove { .. } => "remove", + Request::RemoveAll { .. } => "remove_all", + Request::List { .. } => "list", + Request::RemoveInterface { .. } => "remove_interface", Request::Check { .. } => "check", }; let exchange = async { - let mut stream = UnixStream::connect(socket) - .await - .with_context(|| format!("failed to connect to netd at {}", socket.display()))?; + let mut stream = UnixStream::connect(socket).await.map_err(|error| { + // Only the two errnos that mean "nothing is listening". A socket + // the VMM's user cannot open (`EACCES`, the default `0660` on a + // root-owned socket) or a VMM out of descriptors would otherwise + // read as an absent netd, and every caller that treats absence as + // "nothing to do here" would skip its work at `debug!` -- leaking + // interfaces on a host whose netd is running fine, while telling + // the operator to go start one. + let absent = matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::ConnectionRefused + ); + let error = anyhow::Error::from(error) + .context(format!("failed to connect to netd at {}", socket.display())); + if absent { + error.context(Unreachable) + } else { + error + } + })?; let message = serde_json::to_vec(request)?; if message.len() as u64 > MAX_MESSAGE_SIZE { bail!("netd request is too large"); @@ -154,12 +497,7 @@ pub async fn request(socket: &Path, request: &Request) -> Result Result<()> { None => bind_listener(&config)?, }; info!(address = ?listener.local_addr()?, "netd listening"); + let config = std::sync::Arc::new(config); loop { let (mut stream, _) = listener.accept().await?; - // This timeout bounds async socket reads and writes. handle_request is - // synchronous, so helper execution is bounded separately by - // COMMAND_TIMEOUT rather than preempted by this future timeout. - match timeout(CONNECTION_TIMEOUT, serve_connection(&config, &mut stream)).await { - Ok(Ok(())) => {} - Ok(Err(error)) => warn!(%error, "netd connection failed"), - Err(_) => warn!("netd connection timed out"), - } + // One task per connection, so a request that takes minutes does not + // stop the next one from being *accepted*. + // + // Serialization still holds where it matters, and holds where it is + // actually stated: `handle_request` takes the operation lock, which is + // an flock and blocks between two open descriptions in one process + // just as it does between processes. What a single-connection accept + // loop added on top of that was head-of-line blocking -- a caller + // asking netd a question it answers in microseconds waited for whatever + // netd happened to be doing, and gave up believing netd was not there. + // A collection can run for twenty seconds and a `virsh` for thirty, so + // this was not a corner: it made a busy netd indistinguishable from an + // absent one, and teardown skips an absent netd. + let config = config.clone(); + tokio::spawn(async move { + if let Err(error) = serve_connection(&config, &mut stream).await { + warn!(%error, "netd connection failed"); + } + }); } } @@ -222,107 +572,204 @@ fn bind_listener(config: &NetdConfig) -> Result { Ok(listener) } -async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Result<()> { +/// Serves one connection. +/// +/// The timeouts bound the socket reads and writes and nothing else. They used +/// to wrap the whole exchange, which read as a bound on the request but was +/// not one: `handle_request` is synchronous and shells out, so the timeout +/// could not cancel it -- it only threw away the answer to work that went on +/// running. A caller has its own deadline and will have gone by then; what it +/// cost was netd's own log, which reported "timed out" for a request it in +/// fact completed. Helper execution is bounded where it can be, by +/// COMMAND_TIMEOUT per invocation. +async fn serve_connection( + config: &std::sync::Arc, + stream: &mut UnixStream, +) -> Result<()> { // Access is authorized by the Unix socket's owner, group, and mode. Any // process that can connect is trusted with the complete netd protocol. - let response = match read_request(stream) + let request = timeout(CONNECTION_TIMEOUT, read_request(stream)) .await - .and_then(|request| handle_request(&config.libvirt_uri, request)) - { - Ok((tap, device)) => Response { - ok: true, - tap: Some(tap), - device, - error: None, - }, + .context("timed out reading a netd request")?; + let outcome = match request { + // A peer that connects and closes without sending is asking whether + // anything is listening: netd that died leaves its socket behind. + // Answering that with a parse error and a warning would fill the log + // with reports of it working. + Ok(None) => { + debug!("netd liveness probe"); + return Ok(()); + } + Ok(Some(request)) => { + // `handle_request` shells out to `ip` and `virsh` and waits on the + // operation lock, so it can block for as long as those take. On a + // runtime worker that would stall every other connection's reads + // and writes, which is the head-of-line blocking this daemon just + // stopped having. + let config = config.clone(); + match tokio::task::spawn_blocking(move || handle_request(&config, request)).await { + Ok(outcome) => outcome, + Err(error) => Err(anyhow::anyhow!("netd worker failed: {error}")), + } + } + // A request that arrived but could not be understood still gets an + // answer. A VMM newer than this netd sends operations it does not + // know, and "unknown variant `prepare_foo`" is what tells the operator + // to upgrade; a closed connection tells them nothing. + Err(error) => Err(error), + }; + let response = match outcome { + Ok(outcome) => outcome.into_response(), Err(error) => { warn!(%error, "netd request failed"); Response { ok: false, tap: None, device: None, + queues: None, + removed: None, + interfaces: None, error: Some(format!("{error:#}")), } } }; let encoded = serde_json::to_vec(&response)?; - stream.write_all(&encoded).await?; - stream.shutdown().await?; + timeout(CONNECTION_TIMEOUT, async { + stream.write_all(&encoded).await?; + stream.shutdown().await + }) + .await + .context("timed out answering a netd request")??; Ok(()) } -async fn read_request(stream: &mut UnixStream) -> Result { +/// Reads one request, or `None` if the peer closed without sending anything. +async fn read_request(stream: &mut UnixStream) -> Result> { let mut message = Vec::new(); stream .take(MAX_MESSAGE_SIZE + 1) .read_to_end(&mut message) .await?; + if message.is_empty() { + return Ok(None); + } if message.len() as u64 > MAX_MESSAGE_SIZE { bail!("request exceeds {MAX_MESSAGE_SIZE} bytes"); } - serde_json::from_slice(&message).context("invalid netd request") + serde_json::from_slice(&message) + .map(Some) + .context("invalid netd request") } -fn handle_request(libvirt_uri: &str, request: Request) -> Result<(String, Option)> { +fn handle_request(config: &NetdConfig, request: Request) -> Result { + let libvirt_uri = config.libvirt_uri.as_str(); let _lock = OperationLock::acquire()?; match request { Request::PrepareBridge(request) => { - prepare_bridge(libvirt_uri, &request).map(|tap| (tap, None)) - } - Request::PrepareMacvtap(request) => prepare_macvtap( - libvirt_uri, - &request.identity, - &request.parent, - &request.mac, - request.qemu_uid, - &request.mode, - ) - .map(|(tap, device)| (tap, Some(device))), + prepare_bridge(libvirt_uri, &request, config.filter_policy()) + } + Request::PrepareMacvtap(request) => { + prepare_macvtap(libvirt_uri, &request, config.filter_policy()) + } + Request::List { instance_id } => { + Ok(Outcome::Listed(list_interfaces(libvirt_uri, &instance_id))) + } + Request::RemoveAll { instance_id, vm_id } => { + let removed = sweep_vm_interfaces( + libvirt_uri, + &instance_id, + &vm_id, + config.filter_policy().requires_binding(), + )?; + Ok(Outcome::Swept { removed }) + } + Request::RemoveInterface { tap } => { + if !is_managed_name(&tap) { + bail!("{tap} is not a name netd could have created"); + } + remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; + Ok(Outcome::tap(tap)) + } Request::Remove { identity } => { validate_identity(&identity)?; let tap = tap_name(&identity); - remove_interface(libvirt_uri, &tap)?; - Ok((tap, None)) + // Best effort, whatever the caller says was built. The strict rule + // exists for prepare, where a binding left at the name would block + // the one about to be created; at removal nothing is about to take + // the name, and failing here leaves the interface itself up on the + // bridge rather than just a binding libvirt will hand back on its + // next listing. It also makes this agree with the whole-VM sweep, + // which has always been best effort. + remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; + Ok(Outcome::tap(tap)) } - Request::Check { identity } => { + Request::Check { identity, filtered } => { validate_identity(&identity)?; let tap = tap_name(&identity); if !Path::new("/sys/class/net").join(&tap).exists() { bail!("TAP {tap} does not exist"); } - if !is_macvtap(&tap) { + // An unfiltered TAP has no binding to dump; asking for one would + // report a healthy multiqueue interface as broken. + if filtered && !is_macvtap(&tap) { virsh(libvirt_uri, &["nwfilter-binding-dumpxml", &tap], None)?; } - Ok((tap, None)) + Ok(Outcome::tap(tap)) } } } fn prepare_macvtap( libvirt_uri: &str, - identity: &InterfaceIdentity, - parent: &str, - mac: &str, - qemu_uid: u32, - mode: &str, -) -> Result<(String, String)> { + request: &PrepareMacvtapRequest, + filter: &NetworkFilterConfig, +) -> Result { + let identity = &request.identity; + let parent = request.parent.as_str(); + let qemu_uid = request.qemu_uid; validate_identity(identity)?; validate_name("parent", parent, 15, "_.-")?; if !Path::new("/sys/class/net").join(parent).exists() { bail!("parent interface {parent} does not exist"); } - validate_mac(mac)?; - let mode = if mode.is_empty() { "private" } else { mode }; + // A macvtap parent may be a bridge, and libvirt nwfilter does not apply to + // macvtap. So on a node that requires every bridge TAP to be filtered, a + // macvtap request naming that same bridge is the identical unfiltered L2 + // access the policy exists to refuse, spelled with a different operation. + // An interface enslaved to a bridge reaches the same segment. + if filter.requires_binding() { + let sysfs = Path::new("/sys/class/net").join(parent); + if sysfs.join("bridge").exists() { + bail!("this netd requires filtering, so {parent} may not be a macvtap parent: it is a host bridge"); + } + if sysfs.join("master").exists() { + bail!("this netd requires filtering, so {parent} may not be a macvtap parent: it is enslaved to a bridge"); + } + } + validate_mac(&request.mac)?; + let mac = request.mac.as_str(); + let mode = if request.mode.is_empty() { + "private" + } else { + request.mode.as_str() + }; if !matches!(mode, "private" | "bridge" | "vepa" | "passthru") { bail!("invalid macvtap mode"); } + let queues = validate_queues(request.queues)?; let tap = tap_name(identity); - remove_interface(libvirt_uri, &tap)?; - ip(&[ - "link", "add", "link", parent, "name", &tap, "address", mac, "type", "macvtap", "mode", - mode, - ])?; + remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; + let queue_count = queues.to_string(); + let mut add = vec!["link", "add", "link", parent, "name", &tap, "address", mac]; + if queues > 1 { + // macvtap defaults to a single hardware queue pair. Without this the + // extra tap queues exist but the lower device still serializes. + add.extend_from_slice(&["numtxqueues", &queue_count, "numrxqueues", &queue_count]); + } + add.extend_from_slice(&["type", "macvtap", "mode", mode]); + ip(&add)?; let result = (|| { + set_alias(&tap, identity)?; let ifindex = std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifindex")) .context("failed to read macvtap ifindex")?; @@ -346,11 +793,15 @@ fn prepare_macvtap( })(); match result { Ok(device) => { - info!(%tap, %parent, %mode, %device, "prepared macvtap"); - Ok((tap, device)) + info!(%tap, %parent, %mode, %device, %queues, "prepared macvtap"); + Ok(Outcome::Interface { + tap, + device: Some(device), + queues: Some(queues), + }) } Err(error) => { - let _ = remove_interface(libvirt_uri, &tap); + let _ = remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort); Err(error) } } @@ -384,41 +835,273 @@ impl Drop for OperationLock { } } -fn prepare_bridge(libvirt_uri: &str, request: &PrepareBridgeRequest) -> Result { - validate_prepare_bridge(request)?; +fn prepare_bridge( + libvirt_uri: &str, + request: &PrepareBridgeRequest, + filter: &NetworkFilterConfig, +) -> Result { + validate_prepare_bridge(request, filter)?; + let filtered = request.filtered; let tap = tap_name(&request.identity); // A failed VMM start may leave a deterministic resource behind. Replacing // it makes prepare idempotent without accepting a caller-selected TAP. - remove_interface(libvirt_uri, &tap)?; + // A binding outlives the interface it was bound to and TAP names are + // derived, so the same name comes back: clear whatever is there. Insist + // only when this prepare is about to create a replacement libvirt would + // refuse as a duplicate. + remove_interface( + libvirt_uri, + &tap, + if filtered { + BindingCleanup::Required + } else { + BindingCleanup::BestEffort + }, + )?; let uid = request.qemu_uid.to_string(); - ip(&["tuntap", "add", "dev", &tap, "mode", "tap", "user", &uid])?; + let queues = validate_queues(request.queues)?; + let mut add = vec!["tuntap", "add", "dev", &tap, "mode", "tap"]; + if queues > 1 { + // QEMU refuses to attach when the device's IFF_MULTI_QUEUE state does + // not match its own `queues=` argument, in either direction. + add.push("multi_queue"); + } + add.extend_from_slice(&["user", &uid]); + ip(&add)?; let result = (|| { + // Before anything else it could fail at. An interface that exists + // without a record is one nothing can attribute, and the window in + // which that is true is the window a crash turns permanent. + set_alias(&tap, &request.identity)?; ip(&["link", "set", "dev", &tap, "master", &request.bridge])?; - let xml = binding_xml(request, &tap); - virsh( - libvirt_uri, - &["nwfilter-binding-create", "--validate", "/dev/stdin"], - Some(xml.as_bytes()), - )?; + if filtered { + let xml = binding_xml(request, &tap, filter); + virsh( + libvirt_uri, + &["nwfilter-binding-create", "--validate", "/dev/stdin"], + Some(xml.as_bytes()), + )?; + } ip(&["link", "set", "dev", &tap, "up"])?; Ok(()) })(); if let Err(error) = result { - let _ = remove_interface(libvirt_uri, &tap); + let _ = remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort); return Err(error); } - info!(%tap, bridge = %request.bridge, filter = %request.filter, "prepared filtered TAP"); - Ok(tap) + info!(%tap, bridge = %request.bridge, %filtered, %queues, "prepared TAP"); + Ok(Outcome::Interface { + tap, + device: None, + queues: Some(queues), + // This netd builds interfaces; it is not the host's forwarder. Saying + // nothing here is what tells the caller that, so ports it asked for are + // reported as unmet rather than assumed done. + }) +} + +/// How hard removal must try to clear an nwfilter binding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BindingCleanup { + /// The binding must be gone before this returns, because the caller is + /// about to create one at the same interface name and libvirt refuses a + /// duplicate. + Required, + /// Delete a binding if libvirt can be reached, but do not fail the removal + /// when it cannot. Unfiltered TAPs live on nodes where `libvirtd` need not + /// be running at all, and a stale binding left by an earlier, filtered + /// interface at this name is still worth clearing when it is. + BestEffort, + /// The caller has already decided about the binding. Used by a pass over + /// many interfaces, which asks libvirt once about all of them rather than + /// once per interface. + Skip, +} + +/// Deletes every interface a VM could hold, by deriving each name rather than +/// consulting a record. +/// +/// `validate_identity` caps the NIC index, so the whole space a VM can occupy +/// is enumerable: 256 names, each a `stat` that usually misses. Cleanup is +/// best-effort about unknown bindings on unfiltered nodes, which need not have +/// libvirtd at all. Filtered nodes must confirm their bindings were released. +/// +/// A name with no interface is not skipped. An nwfilter binding outlives the +/// TAP it was bound to, so the one state teardown must not leave behind is +/// exactly the one a `/sys/class/net` check cannot see: the per-name Remove +/// this replaced deleted the binding unconditionally, and a sweep that reaches +/// less than the thing it replaced is not a sweep. Those names are decided +/// against a single listing, because the whole point of enumerating a bounded +/// space is that deciding one name stays cheap. +fn sweep_vm_interfaces( + libvirt_uri: &str, + instance_id: &str, + vm_id: &str, + requires_binding: bool, +) -> Result { + let identity = InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + nic_index: 0, + }; + validate_identity(&identity)?; + let bindings = existing_bindings(libvirt_uri); + // Not `bindings.is_some()`. A listing that could not be produced says + // nothing about whether a *deletion* will work, and reading it as "libvirt + // is down, skip the bindings" would mean a node whose listing breaks for + // any reason silently stops cleaning up bindings at all -- which is worse + // than the per-name asking this listing exists to avoid. The pass finds + // out by trying, once. + let mut libvirt = true; + let mut removed = 0; + let mut first_error = (requires_binding && bindings.is_none()) + .then(|| anyhow::anyhow!("cannot confirm nwfilter cleanup: binding listing failed")); + for nic_index in 0..=MAX_NIC_INDEX { + let tap = tap_name(&InterfaceIdentity { + nic_index, + ..identity.clone() + }); + let present = Path::new("/sys/class/net").join(&tap).exists(); + // A pass gets one answer about libvirt, not one per interface. Asking + // again after it has failed is how a hung `libvirtd` turns a bounded + // collection into an unbounded one. + let known_binding = bindings.as_ref().is_some_and(|held| held.contains(&tap)); + let wanted = present || known_binding; + if libvirt && wanted && !is_macvtap(&tap) { + if let Err(error) = delete_binding(libvirt_uri, &tap) { + warn!(%tap, %error, "failed to remove an nwfilter binding"); + libvirt = false; + // Unfiltered nodes do not require a running libvirtd. An + // unknown, possible binding must not make a successful TAP + // deletion fail there. Still retain failures for bindings we + // actually found, including ones left by an older policy. + if requires_binding || known_binding { + first_error.get_or_insert(error); + } + } else if !present { + info!(%tap, %vm_id, "removed orphaned nwfilter binding"); + } + } + if !libvirt && known_binding { + first_error + .get_or_insert_with(|| anyhow::anyhow!("nwfilter binding {tap} was not released")); + } + if !present { + continue; + } + // Keep going after a failure. Stopping at the first one would leave the + // rest of a VM's interfaces behind over one that is stuck. + match remove_interface(libvirt_uri, &tap, BindingCleanup::Skip) { + Err(error) => { + warn!(%tap, %error, "failed to remove interface"); + first_error.get_or_insert(error); + } + Ok(()) => { + info!(%tap, %vm_id, "removed interface"); + removed += 1; + } + } + } + match first_error { + Some(error) => Err(error).context("failed to remove every interface for this VM"), + None => Ok(removed), + } +} + +/// Every resource netd owns, read off the host rather than out of a record. +/// +/// Ownership is the reserved name plus the kernel's own answer about what kind +/// of device it is; attribution is the interface's alias, checked by +/// re-deriving the name from it. A listing never fails for want of libvirt: on +/// a node that does not filter, `libvirtd` need not be running, and an +/// interface inventory that refused to be produced without it would be +/// unavailable exactly where unfiltered TAPs live. +fn list_interfaces(libvirt_uri: &str, instance_id: &str) -> Vec { + let bindings = existing_bindings(libvirt_uri); + let mut records = Vec::new(); + let mut seen = HashSet::new(); + if let Ok(entries) = std::fs::read_dir("/sys/class/net") { + for entry in entries.flatten() { + let Ok(tap) = entry.file_name().into_string() else { + continue; + }; + if !is_managed_name(&tap) { + continue; + } + let kind = if is_macvtap(&tap) { + "macvtap" + } else if is_tuntap(&tap) { + "tap" + } else { + // The name is netd's to use, but this is not a device netd + // creates. Listing it would invite a caller to delete it. + continue; + }; + let alias = + std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifalias")) + .unwrap_or_default(); + let owner = owner_of(&tap, &alias); + seen.insert(tap.clone()); + records.push(InterfaceRecord { + kind: kind.to_string(), + nic_index: owner.as_ref().map(|identity| identity.nic_index), + instance_id: owner.as_ref().map(|identity| identity.instance_id.clone()), + vm_id: owner.map(|identity| identity.vm_id), + tap, + }); + } + } + // A binding outlives its interface, and an interface is the only thing that + // carries a record, so an orphaned binding can never be attributed. It is + // still netd's: nothing else creates a binding at one of these names. + for name in bindings.into_iter().flatten() { + if !seen.contains(&name) { + records.push(InterfaceRecord { + tap: name, + kind: "binding".to_string(), + instance_id: None, + vm_id: None, + nic_index: None, + }); + } + } + if !instance_id.is_empty() { + records.retain(|record| record.instance_id.as_deref() == Some(instance_id)); + } + records.sort_by(|left, right| left.tap.cmp(&right.tap)); + records } -fn remove_interface(libvirt_uri: &str, tap: &str) -> Result<()> { +fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Result<()> { let macvtap = is_macvtap(tap); if Path::new("/sys/class/net").join(tap).exists() { + // The name is 48 bits of SHA-256, so a collision is not the worry. A + // caller asserting an identity that happens to derive to some + // pre-existing device is: netd runs as root and `ip link delete` does + // not ask what it is deleting. netd creates exactly two kinds of + // device, and the kernel publishes an attribute unique to each. + if !macvtap && !is_tuntap(tap) { + bail!("refusing to delete {tap}: it is neither a tun/tap nor a macvtap device"); + } let _ = ip(&["link", "set", "dev", tap, "down"]); } + // A macvtap interface never carries a binding. Anything else might: this + // name may have been a filtered bridge TAP before, and the binding + // outlives the interface. if !macvtap { - delete_binding(libvirt_uri, tap)?; + match cleanup { + BindingCleanup::Skip => {} + BindingCleanup::Required => delete_binding(libvirt_uri, tap)?, + BindingCleanup::BestEffort => { + // netd refuses to start without virsh, so the binary is always + // here; libvirtd need not be running, and on a node that only + // wants macvtap or multiqueue it usually is not. + if let Err(error) = delete_binding(libvirt_uri, tap) { + warn!(%tap, "could not clear a possible nwfilter binding: {error:#}"); + } + } + } } if Path::new("/sys/class/net").join(tap).exists() { ip(&["link", "delete", "dev", tap])?; @@ -427,6 +1110,24 @@ fn remove_interface(libvirt_uri: &str, tap: &str) -> Result<()> { Ok(()) } +/// Records who an interface belongs to, on the interface. See +/// [`interface_alias`]. +fn set_alias(tap: &str, identity: &InterfaceIdentity) -> Result<()> { + let alias = interface_alias(identity); + ip(&["link", "set", "dev", tap, "alias", &alias]) + .with_context(|| format!("failed to record ownership on {tap}")) +} + +/// Whether this is a tun/tap device. `tun_flags` is published by the tun +/// driver and by nothing else, so its presence is the kernel's own answer -- +/// as `macvtap/` is for the other kind of device netd creates. +fn is_tuntap(interface: &str) -> bool { + Path::new("/sys/class/net") + .join(interface) + .join("tun_flags") + .exists() +} + fn is_macvtap(interface: &str) -> bool { Path::new("/sys/class/net") .join(interface) @@ -434,32 +1135,37 @@ fn is_macvtap(interface: &str) -> bool { .exists() } +/// Deletes an interface's nwfilter binding, if it has one. +/// +/// Goes through the same `COMMAND_TIMEOUT`-bounded helper as every other virsh +/// call. Every mutating operation holds the operation lock, so an unbounded +/// call here would let one unreachable libvirt stall every other VM's prepare +/// and remove behind it. fn delete_binding(uri: &str, tap: &str) -> Result<()> { - let output = Command::new(VIRSH_PATH) - .args(["--connect", uri, "nwfilter-binding-delete", tap]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .context("failed to execute virsh")?; - if output.status.success() { - return Ok(()); - } - let error = String::from_utf8_lossy(&output.stderr); - if error.contains("Network filter binding not found") { - return Ok(()); + match virsh(uri, &["nwfilter-binding-delete", tap], None) { + Ok(()) => Ok(()), + // Removal is idempotent. Having no binding is the normal case for + // macvtap, for unfiltered multiqueue TAPs, and for any name being + // reused after an earlier removal already cleared it. + Err(error) + if error + .to_string() + .contains("Network filter binding not found") => + { + Ok(()) + } + Err(error) => Err(error).context(format!("virsh failed to delete binding {tap}")), } - bail!("virsh failed to delete binding {tap}: {}", error.trim()) } -fn binding_xml(request: &PrepareBridgeRequest, tap: &str) -> String { +fn binding_xml(request: &PrepareBridgeRequest, tap: &str, filter: &NetworkFilterConfig) -> String { let owner_uuid = stable_uuid(&request.identity); let owner_name = format!( "dstack:{}:{}:{}", request.identity.instance_id, request.identity.vm_id, request.identity.nic_index ); let mut parameters = String::new(); - for (name, value) in &request.parameters { + for (name, value) in &filter.parameters { parameters.push_str(&format!( "", xml_escape(name), @@ -474,7 +1180,7 @@ fn binding_xml(request: &PrepareBridgeRequest, tap: &str) -> String { owner_uuid, xml_escape(tap), xml_escape(&request.mac), - xml_escape(&request.filter), + xml_escape(&filter.filter), parameters ) } @@ -494,9 +1200,24 @@ fn stable_uuid(identity: &InterfaceIdentity) -> Uuid { Uuid::from_bytes(bytes) } -fn validate_prepare_bridge(request: &PrepareBridgeRequest) -> Result<()> { +fn validate_prepare_bridge( + request: &PrepareBridgeRequest, + filter: &NetworkFilterConfig, +) -> Result<()> { validate_identity(&request.identity)?; validate_name("bridge", &request.bridge, 15, "_.-")?; + // Unfiltered bridge TAPs exist for unfiltered multiqueue, and netd holds + // that policy itself rather than trusting the caller with it. netd is the + // privileged side of this socket; on a node configured to filter bridge + // traffic, "build me a TAP on br0 with no nwfilter binding" is precisely + // the request the boundary exists to refuse, and anything that can reach + // the socket can make it. + // + // Refused before the host is inspected: this is about the request, not + // about what happens to exist on this machine. + if filter.requires_binding() && !request.filtered { + bail!("this netd requires an nwfilter binding on every bridge TAP"); + } if !Path::new("/sys/class/net") .join(&request.bridge) .join("bridge") @@ -505,16 +1226,6 @@ fn validate_prepare_bridge(request: &PrepareBridgeRequest) -> Result<()> { bail!("{} is not a host bridge", request.bridge); } validate_mac(&request.mac)?; - validate_name("filter", &request.filter, 128, "_.:-")?; - if request.parameters.len() > 64 { - bail!("too many nwfilter parameters"); - } - for (name, value) in &request.parameters { - validate_name("parameter name", name, 64, "_")?; - if value.len() > 512 || value.contains('\0') { - bail!("invalid nwfilter parameter value"); - } - } Ok(()) } @@ -527,15 +1238,46 @@ fn validate_identity(identity: &InterfaceIdentity) -> Result<()> { bail!("invalid {label}"); } } - if identity.nic_index > 255 { + if identity.nic_index > MAX_NIC_INDEX { bail!("NIC index is out of range"); } + // An identity that cannot be recorded on the interface is refused rather + // than built unattributed. A host resource nothing can name the owner of + // is the thing this whole path exists to stop producing, and the kernel's + // alias is the only place with the interface's exact lifetime to put it. + let alias = interface_alias(identity); + if alias.len() > MAX_IFALIAS { + bail!( + "identity is too long to record on the interface: {} bytes of {MAX_IFALIAS}", + alias.len() + ); + } + // The record puts the two free-form fields last, so only the first of them + // has to be unambiguous. + if identity.instance_id.contains(':') { + bail!("instance ID must not contain ':'"); + } Ok(()) } +/// Normalizes a requested queue pair count. Zero means the caller did not ask +/// for multiqueue, which is the same device shape as one queue pair. +fn validate_queues(queues: u32) -> Result { + if queues > MAX_QUEUES { + bail!("queues must not exceed {MAX_QUEUES}"); + } + Ok(queues.max(1)) +} + fn validate_name(label: &str, value: &str, max: usize, punctuation: &str) -> Result<()> { if value.is_empty() || value.len() > max + // `.` and `..` pass the charset check below, and every name validated + // here is then joined onto a sysfs path to ask whether the interface + // exists. `/sys/class/net/..` exists, so the question would be answered + // about a directory rather than about an interface. + || value == "." + || value == ".." || !value .chars() .all(|ch| ch.is_ascii_alphanumeric() || punctuation.contains(ch)) @@ -575,13 +1317,47 @@ fn ip(args: &[&str]) -> Result<()> { } fn virsh(uri: &str, args: &[&str], stdin: Option<&[u8]>) -> Result<()> { + virsh_output(uri, args, stdin).map(|_| ()) +} + +fn virsh_output(uri: &str, args: &[&str], stdin: Option<&[u8]>) -> Result { let mut full_args = vec!["--connect", uri]; full_args.extend_from_slice(args); - run_command(VIRSH_PATH, &full_args, stdin) + run_command_with_timeout(VIRSH_PATH, &full_args, stdin, COMMAND_TIMEOUT) +} + +/// Every nwfilter binding libvirt holds at a name netd could have created. +/// +/// One call, so that a sweep can decide 256 names against a set instead of +/// asking libvirt 256 times. `None` means libvirt could not be asked at all, +/// which on a node running unfiltered TAPs is the normal state -- `virsh` must +/// be installed for netd to start, but `libvirtd` need not be running. +/// +/// The command has no machine-readable mode: it prints a two-line header and +/// then one binding per line, interface name first, and it accepts no options +/// at all -- `--name` is not one of them, and asking for it fails the whole +/// call. Narrowing to netd's own name space is what makes parsing a human +/// table safe: a header, a rule line, or a column that moves cannot produce a +/// `dt` name, and a binding at any other name is not netd's to reason about. +fn existing_bindings(uri: &str) -> Option> { + match virsh_output(uri, &["nwfilter-binding-list"], None) { + Ok(output) => Some( + output + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|name| is_managed_name(name)) + .map(str::to_string) + .collect(), + ), + Err(error) => { + debug!("could not list nwfilter bindings: {error:#}"); + None + } + } } fn run_command(program: &str, args: &[&str], stdin: Option<&[u8]>) -> Result<()> { - run_command_with_timeout(program, args, stdin, COMMAND_TIMEOUT) + run_command_with_timeout(program, args, stdin, COMMAND_TIMEOUT).map(|_| ()) } fn run_command_with_timeout( @@ -589,7 +1365,7 @@ fn run_command_with_timeout( args: &[&str], stdin: Option<&[u8]>, command_timeout: Duration, -) -> Result<()> { +) -> Result { let mut child = Command::new(program) .args(args) .stdin(if stdin.is_some() { @@ -621,7 +1397,7 @@ fn run_command_with_timeout( let error = String::from_utf8_lossy(&output.stderr); bail!("{} failed: {}", Path::new(program).display(), error.trim()); } - Ok(()) + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } fn require_executable(path: &str) -> Result<()> { @@ -645,9 +1421,167 @@ fn prepare_socket_path(socket: &Path) -> Result<()> { Ok(()) } +/// A netd that exists only to be talked to. +/// +/// The VMM's side of this protocol -- what it falls back to, what it refuses, +/// what it does when the answer is missing -- had no test at all, because +/// every path needed a privileged daemon. It does not: it needs something that +/// answers on a socket. This is that, scripted per behaviour, recording what +/// it was asked so a test can assert on the conversation rather than on its +/// effects. +#[cfg(test)] +pub(crate) mod testing { + use std::{ + path::{Path, PathBuf}, + sync::{Arc, Mutex}, + }; + + use serde_json::{json, Value}; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::UnixListener, + }; + + /// How the fake answers. + #[derive(Debug, Clone)] + pub(crate) enum Behavior { + /// Answers every listed operation with a plausible success, and + /// anything else with the error `serde` produces for an unknown one. + Handles(Vec), + /// An older netd: every operation this one added is an error. + Legacy, + } + + impl Behavior { + pub(crate) fn handling(operations: &[&str]) -> Self { + Self::Handles(operations.iter().map(|name| name.to_string()).collect()) + } + } + + pub(crate) struct FakeNetd { + _dir: tempfile::TempDir, + socket: PathBuf, + seen: Arc>>, + } + + impl FakeNetd { + pub(crate) fn spawn(behavior: Behavior) -> Self { + Self::spawn_holding(behavior, Vec::new()) + } + + /// A netd that holds these interfaces, whatever else it does. + pub(crate) fn spawn_holding(behavior: Behavior, interfaces: Vec) -> Self { + Self::start(behavior, interfaces) + } + + fn start(behavior: Behavior, interfaces: Vec) -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("netd.sock"); + let listener = UnixListener::bind(&socket).expect("bind"); + let seen = Arc::new(Mutex::new(Vec::new())); + let recorder = seen.clone(); + let interfaces = std::sync::Arc::new(interfaces); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let behavior = behavior.clone(); + let interfaces = interfaces.clone(); + let recorder = recorder.clone(); + tokio::spawn(async move { + let mut message = Vec::new(); + if stream.read_to_end(&mut message).await.is_err() { + return; + } + let Ok(request) = serde_json::from_slice::(&message) else { + return; + }; + recorder.lock().expect("poisoned").push(request.clone()); + let response = answer(&behavior, &interfaces, &request); + let _ = stream + .write_all(&serde_json::to_vec(&response).unwrap()) + .await; + let _ = stream.shutdown().await; + }); + } + }); + Self { + _dir: dir, + socket, + seen, + } + } + + pub(crate) fn socket(&self) -> &Path { + &self.socket + } + + /// Every request it was sent, in order. + pub(crate) fn seen(&self) -> Vec { + self.seen.lock().expect("poisoned").clone() + } + + pub(crate) fn operations(&self) -> Vec { + self.seen() + .iter() + .map(|request| request["operation"].as_str().unwrap_or("?").to_string()) + .collect() + } + } + + fn answer(behavior: &Behavior, interfaces: &[Value], request: &Value) -> Value { + let operation = request["operation"].as_str().unwrap_or_default(); + let operations = match behavior { + Behavior::Legacy => { + return match operation { + // What the real thing answers for an operation it knows. + "prepare_bridge" | "prepare_macvtap" | "remove" | "check" => { + json!({"ok": true, "tap": "dtdeadbeef00"}) + } + other => json!({ + "ok": false, + "error": format!("invalid netd request: unknown variant `{other}`"), + }), + }; + } + Behavior::Handles(operations) => operations, + }; + if !operations.iter().any(|name| name == operation) { + return json!({ + "ok": false, + "error": format!("invalid netd request: unknown variant `{operation}`"), + }); + } + match operation { + "prepare_bridge" | "prepare_macvtap" => json!({ + "ok": true, + "tap": "dtdeadbeef00", + "queues": request["queues"].as_u64().unwrap_or(1).max(1), + }), + "remove" | "check" => json!({"ok": true, "tap": "dtdeadbeef00"}), + "remove_all" => json!({"ok": true, "removed": 0, "incomplete": false}), + "list" => { + let instance = request["instance_id"].as_str().unwrap_or_default(); + let held: Vec = interfaces + .iter() + .filter(|record| { + instance.is_empty() || record["instance_id"].as_str() == Some(instance) + }) + .cloned() + .collect(); + json!({"ok": true, "interfaces": held}) + } + "remove_interface" => json!({"ok": true, "tap": request["tap"]}), + _ => json!({"ok": true}), + } + } +} + #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeMap; fn identity(instance: &str, vm: &str, nic_index: usize) -> InterfaceIdentity { InterfaceIdentity { @@ -673,10 +1607,16 @@ mod tests { bridge: "br0".into(), mac: "02:00:00:00:00:01".into(), qemu_uid: 1000, + filtered: true, + queues: 0, + workdir: String::new(), + }; + let filter = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, filter: "clean-traffic".into(), parameters: BTreeMap::from([("IP".into(), "10.0.0.2<&".into())]), }; - let xml = binding_xml(&request, "dt123"); + let xml = binding_xml(&request, "dt123", &filter); assert!(xml.contains("instance<&")); assert!(xml.contains("10.0.0.2<&")); assert!(!xml.contains("instance<&")); @@ -685,6 +1625,10 @@ mod tests { #[test] fn validation_rejects_injected_host_names() { assert!(validate_name("bridge", "br0;id", 15, "_.-").is_err()); + // `/sys/class/net/..` exists, so an existence check on this name would + // answer about a directory rather than about an interface. + assert!(validate_name("parent", "..", 15, "_.-").is_err()); + assert!(validate_name("parent", ".", 15, "_.-").is_err()); assert!(validate_name("filter", "../../filter", 128, "_.:-").is_err()); assert!(validate_mac("ff:ff:ff:ff:ff:ff").is_err()); } @@ -709,8 +1653,9 @@ mod tests { bridge: "br0".into(), mac: "02:00:00:00:00:01".into(), qemu_uid: 1000, - filter: "clean-traffic".into(), - parameters: BTreeMap::new(), + filtered: true, + queues: 0, + workdir: String::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -719,6 +1664,47 @@ mod tests { assert!(value.get("identity").is_none()); } + #[test] + fn queue_counts_normalize_to_at_least_one_and_stay_bounded() { + assert_eq!(validate_queues(0).unwrap(), 1); + assert_eq!(validate_queues(1).unwrap(), 1); + assert_eq!(validate_queues(MAX_QUEUES).unwrap(), MAX_QUEUES); + assert!(validate_queues(MAX_QUEUES + 1).is_err()); + } + + #[test] + fn queue_count_travels_with_the_prepare_request() { + let request = Request::PrepareBridge(PrepareBridgeRequest { + identity: identity("instance", "vm", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: false, + queues: 4, + workdir: String::new(), + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["queues"], 4); + assert_eq!(value["filtered"], false); + + // QEMU refuses a device whose IFF_MULTI_QUEUE state disagrees with its + // own `queues=`, so a request that leaves the count to netd's + // imagination is one netd must not answer. + let error = serde_json::from_value::(serde_json::json!({ + "operation": "prepare_bridge", + "instance_id": "instance", + "vm_id": "vm", + "nic_index": 0, + "bridge": "br0", + "mac": "02:00:00:00:00:01", + "qemu_uid": 1000, + "filtered": true, + })) + .unwrap_err(); + assert!(error.to_string().contains("queues"), "{error}"); + assert_eq!(validate_queues(0).unwrap(), 1); + } + #[test] fn macvtap_prepare_has_a_dedicated_operation() { let request = Request::PrepareMacvtap(PrepareMacvtapRequest { @@ -727,6 +1713,8 @@ mod tests { mac: "02:00:00:00:00:01".into(), qemu_uid: 1000, mode: "private".into(), + queues: 0, + workdir: String::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_macvtap"); @@ -742,8 +1730,9 @@ mod tests { bridge: "br0".into(), mac: "02:00:00:00:00:01".into(), qemu_uid: 1000, - filter: "clean-traffic".into(), - parameters: BTreeMap::new(), + filtered: true, + queues: 0, + workdir: String::new(), }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -752,20 +1741,162 @@ mod tests { assert!(value.get("identity").is_none()); } + /// Connecting and closing without sending is how the VMM checks that netd + /// is alive, because a netd that died leaves its socket behind. It has to + /// be handled promptly, and quietly: the VMM does it once per status query + /// that mentions a stopped VM, and netd's accept loop is serialized, so + /// treating a probe as a failed request would both fill the log and put + /// noise in front of real work. #[tokio::test] - async fn disconnected_client_is_confined_to_one_connection() { + async fn a_connection_that_sends_nothing_is_a_liveness_probe() { let (mut server, client) = UnixStream::pair().unwrap(); drop(client); let result = timeout( Duration::from_secs(1), - serve_connection(&NetdConfig::default(), &mut server), + serve_connection(&std::sync::Arc::new(NetdConfig::default()), &mut server), ) .await; assert!(result.is_ok(), "disconnected peer blocked the handler"); - // Either the EOF is reported while reading or the response write sees - // EPIPE. In both cases serve() logs this per-connection error and keeps - // accepting clients. - assert!(result.unwrap().is_err()); + assert!(result.unwrap().is_ok(), "a probe is not a failed request"); + } + + /// Only an empty connection is a probe. A peer that does send something, + /// and sends nonsense, is still a request -- and still gets an answer it + /// can read, which is how a VMM newer than its netd learns to say so. + #[tokio::test] + async fn a_request_that_cannot_be_understood_still_gets_an_answer() { + let (mut server, client) = UnixStream::pair().unwrap(); + drop(client); + assert!(read_request(&mut server).await.unwrap().is_none()); + + let (mut server, mut client) = UnixStream::pair().unwrap(); + // An operation only a newer VMM knows about. + client + .write_all(br#"{"operation":"prepare_something_new"}"#) + .await + .unwrap(); + client.shutdown().await.unwrap(); + serve_connection(&std::sync::Arc::new(NetdConfig::default()), &mut server) + .await + .unwrap(); + + let mut reply = Vec::new(); + client.read_to_end(&mut reply).await.unwrap(); + let reply: serde_json::Value = serde_json::from_slice(&reply).unwrap(); + assert_eq!(reply["ok"], false); + assert!( + reply["error"].as_str().unwrap().contains("unknown variant"), + "{reply}" + ); + } + + /// netd is the privileged side of this socket. "Build me a TAP on br0 with + /// no nwfilter binding" is the request the boundary exists to refuse on a + /// filtering node, and before this the daemon simply did what it was told, + /// leaving the invariant with the unprivileged caller. + #[test] + fn a_filtering_node_refuses_an_unfiltered_bridge_tap() { + let request = PrepareBridgeRequest { + identity: InterfaceIdentity { + instance_id: "i".into(), + vm_id: "v".into(), + nic_index: 0, + }, + // A name no host has, so the check after this one is the one that + // fails when this one does not. + bridge: "dstack-nobr0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: false, + queues: 4, + workdir: String::new(), + }; + let filtering = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, + ..NetworkFilterConfig::default() + }; + let error = validate_prepare_bridge(&request, &filtering).unwrap_err(); + assert!( + error.to_string().contains("requires an nwfilter binding"), + "{error}" + ); + + // An unfiltered node still builds them; that is what multiqueue needs. + // It gets as far as asking the host about the bridge, which is the + // next check and not this one's business. + let error = validate_prepare_bridge(&request, &NetworkFilterConfig::default()).unwrap_err(); + assert!( + error.to_string().contains("is not a host bridge"), + "{error}" + ); + } + + /// nwfilter does not apply to macvtap, and a macvtap parent may be the very + /// bridge the policy protects. Refusing an unfiltered bridge TAP while + /// handing out a macvtap on the same segment would leave the policy + /// enforced only against the spelling that happens to be checked. + #[test] + fn a_filtering_node_refuses_a_macvtap_parent_that_is_a_bridge() { + let filtering = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, + ..NetworkFilterConfig::default() + }; + let bridges: Vec = std::fs::read_dir("/sys/class/net") + .into_iter() + .flatten() + .flatten() + .filter(|entry| entry.path().join("bridge").exists()) + .filter_map(|entry| entry.file_name().into_string().ok()) + .collect(); + let Some(bridge) = bridges.first() else { + // Nothing to assert against on a host with no bridge; the unit + // below still pins the enslaved case's sysfs predicate. + return; + }; + let request = PrepareMacvtapRequest { + identity: identity("i", "v", 0), + parent: bridge.clone(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + mode: "bridge".into(), + queues: 4, + workdir: String::new(), + }; + let error = match prepare_macvtap("test:///default", &request, &filtering) { + Err(error) => error, + Ok(_) => panic!("a filtering node must not build a macvtap on a host bridge"), + }; + assert!(error.to_string().contains("is a host bridge"), "{error}"); + } + + /// The request says whether to bind a filter, never which one. `allow-arp` + /// contains no drop rule at all, and `clean-traffic` pinned to the + /// gateway's MAC and IP through its parameters filters nothing useful + /// either -- both would satisfy a policy that only asked for "some filter". + #[test] + fn the_bound_filter_comes_from_netds_own_configuration() { + let request = PrepareBridgeRequest { + identity: identity("i", "v", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: true, + queues: 1, + workdir: String::new(), + }; + // Nothing on the wire can name a filter: the field does not exist. + let wire = serde_json::to_value(Request::PrepareBridge(request.clone())).unwrap(); + assert!(wire.get("filter").is_none(), "{wire}"); + assert!(wire.get("parameters").is_none(), "{wire}"); + + let policy = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, + filter: "clean-traffic".into(), + parameters: BTreeMap::from([("IP".into(), "10.0.0.2".into())]), + }; + let xml = binding_xml(&request, "dt123", &policy); + assert!(xml.contains("filter='clean-traffic'"), "{xml}"); + assert!(xml.contains("value='10.0.0.2'"), "{xml}"); } #[test] @@ -781,4 +1912,357 @@ mod tests { assert!(error.to_string().contains("timed out")); assert!(started.elapsed() < Duration::from_secs(2)); } + + #[test] + fn the_workdir_travels_but_older_callers_may_omit_it() { + let request = Request::PrepareBridge(PrepareBridgeRequest { + identity: identity("instance", "vm", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: true, + queues: 1, + workdir: "/opt/dstack/run/vm/vm".into(), + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["workdir"], "/opt/dstack/run/vm/vm"); + // It is a log line, not an input, so a caller that never sets it is not + // asking for anything different. + let Request::PrepareBridge(decoded) = decode_minimal_bridge() else { + panic!("expected a bridge prepare"); + }; + assert_eq!(decoded.workdir, ""); + } + + /// A prepare carrying only the fields that predate this change. + fn decode_minimal_bridge() -> Request { + serde_json::from_value(serde_json::json!({ + "operation": "prepare_bridge", + "instance_id": "instance", + "vm_id": "vm", + "nic_index": 0, + "bridge": "br0", + "mac": "02:00:00:00:00:01", + "qemu_uid": 1000, + "filtered": true, + "queues": 1, + })) + .unwrap() + } + + #[test] + fn a_whole_vm_sweep_needs_no_record_of_what_it_is_deleting() { + let value = serde_json::to_value(Request::RemoveAll { + instance_id: "instance".into(), + vm_id: "vm".into(), + }) + .unwrap(); + assert_eq!(value["operation"], "remove_all"); + assert_eq!(value["instance_id"], "instance"); + assert_eq!(value["vm_id"], "vm"); + // No NIC index: the point is reaching the ones the caller can no longer + // name, so it names none and netd derives the whole space instead. + assert!(value.get("nic_index").is_none()); + + // That space is bounded by what an identity may say, which is what + // makes deriving it cheap enough to do on every launch. + let mut identity = identity("instance", "vm", MAX_NIC_INDEX); + assert!(validate_identity(&identity).is_ok()); + identity.nic_index = MAX_NIC_INDEX + 1; + assert!(validate_identity(&identity).is_err()); + } + + /// A sweep names no single interface, so its answer must not carry an + /// empty one. `request()` reads a blank `tap` as a malformed response, and + /// a third-party netd copying this shape would have to send a field that + /// means nothing. + #[test] + fn a_sweep_reports_a_count_and_no_interface() { + let value = serde_json::to_value(Outcome::Swept { removed: 3 }.into_response()).unwrap(); + assert_eq!(value["removed"], 3); + assert!(value.get("tap").is_none()); + + // A prepare still names one, because the caller hands that name to QEMU. + let value = serde_json::to_value(Outcome::tap("dtabc".into()).into_response()).unwrap(); + assert_eq!(value["tap"], "dtabc"); + assert!(value.get("removed").is_none()); + } + + /// The record is a hint; the name is the proof. Everything that can go + /// wrong with reading a string off an interface -- forged, truncated, + /// ambiguous, absent -- has to land in the same bucket, and it has to be + /// the bucket a collection treats conservatively. + #[test] + fn an_interface_says_whose_it_is_and_the_name_is_what_proves_it() { + let nic = identity("path-abc", "vm-1", 3); + let tap = tap_name(&nic); + let alias = interface_alias(&nic); + assert_eq!(alias, "dstack1:3:path-abc:vm-1"); + + let owner = owner_of(&tap, &alias).expect("its own record checks out"); + assert_eq!(owner.instance_id, "path-abc"); + assert_eq!(owner.vm_id, "vm-1"); + assert_eq!(owner.nic_index, 3); + + // A record naming some other interface proves nothing about this one. + // This is what makes the record unforgeable without making it + // authoritative: anything that can reach the socket can write a + // string, but only the true identity re-derives the name. + let forged = interface_alias(&identity("path-abc", "someone-elses-vm", 3)); + assert!(owner_of(&tap, &forged).is_none()); + assert!(owner_of(&tap, "").is_none()); + assert!(owner_of(&tap, "dstack1:3:path-abc").is_none()); + assert!(owner_of(&tap, &alias[..alias.len() - 2]).is_none()); + // A format this build does not know is not this format. + assert!(owner_of(&tap, &alias.replace("dstack1", "dstack2")).is_none()); + + // The two free-form fields are last and only the first of them has to + // be unambiguous, so a VM ID carrying the separator still reads back. + let odd = identity("path-abc", "vm:with:colons", 0); + assert_eq!( + owner_of(&tap_name(&odd), &interface_alias(&odd)).map(|owner| owner.vm_id), + Some("vm:with:colons".to_string()) + ); + // An instance ID carrying it is refused instead of mis-parsed. + assert!(validate_identity(&identity("path:abc", "vm-1", 0)).is_err()); + } + + /// An identity that cannot be recorded would produce an interface nothing + /// can attribute, which is the state this whole path exists to stop + /// creating. Refusing it is the only answer that keeps the invariant. + #[test] + fn an_identity_too_long_to_record_is_refused() { + let long = "v".repeat(128); + assert!(validate_identity(&identity("instance", &long, 0)).is_ok()); + let identity_too_long = identity(&"i".repeat(128), &long, 255); + assert!(interface_alias(&identity_too_long).len() > MAX_IFALIAS); + let error = validate_identity(&identity_too_long) + .unwrap_err() + .to_string(); + assert!(error.contains("too long to record"), "{error}"); + } + + /// The name space netd claims. A collection deletes what matches, so what + /// matches has to be exactly what netd can produce. + #[test] + fn the_managed_name_space_is_exactly_what_netd_produces() { + assert!(is_managed_name(&tap_name(&identity("instance", "vm", 0)))); + assert!(is_managed_name("dt0123456789ab")); + assert!(!is_managed_name("dt0123456789AB"), "digests are lower case"); + assert!(!is_managed_name("dt0123456789a"), "one short"); + assert!(!is_managed_name("dt0123456789abc"), "one long"); + assert!(!is_managed_name("dtzzzzzzzzzzzz")); + assert!(!is_managed_name("virbr0")); + assert!(!is_managed_name("eth0")); + // The whole space fits in IFNAMSIZ, or the kernel would refuse the + // names this reserves. + assert!(tap_name(&identity("instance", "vm", 255)).len() < 16); + } + + /// The command prints a table for a human and accepts no options to make it + /// print anything else, so this parses one. Narrowing to netd's own name + /// space is what makes that safe. + #[test] + fn the_binding_listing_reads_a_table_meant_for_a_person() { + let output = "\ + Port Dev Filter +--------------------------------- + dt1e053266e9f7 clean-traffic + dt28b105b3031a clean-traffic + vnet3 some-other-filter +"; + let names: HashSet = output + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|name| is_managed_name(name)) + .map(str::to_string) + .collect(); + assert_eq!(names.len(), 2); + assert!(names.contains("dt1e053266e9f7")); + // The header, the rule, and a binding that is not netd's all fall out. + assert!(!names.contains("Port")); + assert!(!names.contains("vnet3")); + } + + /// Everything else here reasons about strings. This puts the reasoning + /// next to the kernel: that an alias survives on a device netd actually + /// creates, that enumeration finds it, that the guards refuse what they + /// are meant to, and that removal leaves nothing. + /// + /// Refuses to run in the host's network namespace, so it cannot touch a + /// real node's interfaces even when it fails. Unsharing one from inside + /// the test is not enough: `/sys/class/net` keeps showing the old + /// namespace until sysfs is remounted, which is most of what `ip netns + /// exec` does. So it asks to be put in one: + /// + /// ```text + /// cargo test -p dstack-vmm --bins --no-run + /// sudo ip netns add dstack-netd-test + /// sudo ip netns exec dstack-netd-test \ + /// target/debug/deps/dstack_vmm- --ignored --test-threads=1 + /// sudo ip netns del dstack-netd-test + /// ``` + #[test] + #[ignore = "needs root and its own network namespace; see the doc comment"] + fn a_real_interface_carries_its_record_and_removal_leaves_nothing() { + assert!( + nix::unistd::Uid::effective().is_root(), + "this test needs root" + ); + let (mine, init) = ( + std::fs::read_link("/proc/self/ns/net").unwrap(), + std::fs::read_link("/proc/1/ns/net").unwrap(), + ); + assert_ne!( + mine, init, + "run this inside its own network namespace; it creates and deletes interfaces" + ); + // Nothing in this namespace to talk to, which is also the state of a + // node that does not filter: the listing has to work without libvirt. + let uri = "qemu:///nonexistent-for-this-test"; + + let nic = identity("test-instance", "vm-1", 2); + let tap = tap_name(&nic); + ip(&["tuntap", "add", "dev", &tap, "mode", "tap"]).unwrap(); + set_alias(&tap, &nic).unwrap(); + assert!(is_tuntap(&tap), "the kernel publishes tun_flags for a TAP"); + + let records = list_interfaces(uri, ""); + let record = records + .iter() + .find(|record| record.tap == tap) + .expect("an interface netd created is one netd can find"); + assert_eq!(record.instance_id.as_deref(), Some("test-instance")); + assert_eq!(record.vm_id.as_deref(), Some("vm-1")); + assert_eq!(record.nic_index, Some(2)); + assert_eq!(record.kind, "tap"); + // Narrowing by instance is what keeps one VMM's collection off + // another's interfaces. + assert_eq!(list_interfaces(uri, "test-instance").len(), 1); + assert!(list_interfaces(uri, "someone-else").is_empty()); + + // A device with one of netd's names that netd did not create. The name + // is 48 bits of digest, so this is not about collisions -- it is that + // `ip link delete` does not ask what it is deleting, and netd runs as + // root. + let impostor = tap_name(&identity("test-instance", "not-a-tap", 0)); + ip(&["link", "add", &impostor, "type", "dummy"]).unwrap(); + assert!(is_managed_name(&impostor)); + assert!( + !list_interfaces(uri, "") + .iter() + .any(|record| record.tap == impostor), + "a device netd did not create is not offered up for collection" + ); + let refused = remove_interface(uri, &impostor, BindingCleanup::Skip).unwrap_err(); + assert!(refused.to_string().contains("refusing to delete")); + + // An interface whose record does not re-derive its own name proves + // nothing, and lands in the same bucket as no record at all. + ip(&[ + "link", + "set", + "dev", + &tap, + "alias", + "dstack1:2:test-instance:some-other-vm", + ]) + .unwrap(); + let records = list_interfaces(uri, ""); + let record = records.iter().find(|record| record.tap == tap).unwrap(); + assert!(record.instance_id.is_none(), "a forged record is no record"); + + remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); + assert!(!Path::new("/sys/class/net").join(&tap).exists()); + assert!(!list_interfaces(uri, "") + .iter() + .any(|record| record.tap == tap)); + // Removing what is not there is not an error: a sweep derives names + // and most of them miss. + remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); + } + /// Run with the same isolated-network-namespace setup as the test above. + #[test] + #[ignore = "needs root and its own network namespace"] + fn sweeps_without_libvirt_follow_the_nodes_filter_policy() { + assert!(nix::unistd::Uid::effective().is_root()); + assert_ne!( + std::fs::read_link("/proc/self/ns/net").unwrap(), + std::fs::read_link("/proc/1/ns/net").unwrap(), + "run this inside its own network namespace", + ); + let uri = "qemu:///nonexistent-for-this-test"; + for requires_binding in [false, true] { + let nic = identity("sweep-test", "vm-1", 0); + let tap = tap_name(&nic); + ip(&["tuntap", "add", "dev", &tap, "mode", "tap"]).unwrap(); + let mut config = NetdConfig { + libvirt_uri: uri.into(), + ..Default::default() + }; + config.network_filter = Some(NetworkFilterConfig { + mode: if requires_binding { + crate::config::NetworkFilterMode::Libvirt + } else { + crate::config::NetworkFilterMode::None + }, + ..Default::default() + }); + let sweep = || { + handle_request( + &config, + Request::RemoveAll { + instance_id: nic.instance_id.clone(), + vm_id: nic.vm_id.clone(), + }, + ) + }; + let result = sweep(); + // Even a binding failure must not leave the TAP on the bridge. + assert!(!Path::new("/sys/class/net").join(&tap).exists()); + if requires_binding { + assert!(result.is_err()); + // A retry cannot claim success just because the TAP is gone: + // its binding may still exist in the unreachable libvirt. + assert!(sweep().is_err()); + } else { + assert!(matches!(result.unwrap(), Outcome::Swept { removed: 1 })); + assert!(matches!(sweep().unwrap(), Outcome::Swept { removed: 0 })); + } + } + } + + /// A netd that answers a sweep with no count did not sweep. Reading the + /// absent field as zero is the same conflation `queues` is shaped to + /// avoid, and here it would report a netd that cannot collect a VM's + /// interfaces as a VM that had none. + #[tokio::test] + async fn a_sweep_without_a_count_is_not_read_as_an_empty_one() { + // Claims the operation, answers without the field. + let netd = testing::FakeNetd::spawn(testing::Behavior::handling(&[])); + let error = remove_all(netd.socket(), "instance", "vm") + .await + .expect_err("an answer with no count is not a successful sweep"); + assert!(!is_unreachable(&error)); + + let netd = testing::FakeNetd::spawn(testing::Behavior::handling(&["remove_all"])); + let removed = remove_all(netd.socket(), "instance", "vm").await.unwrap(); + assert_eq!(removed, 0); + } + + /// Every "an unreachable netd is not a failure" branch in the VMM hangs + /// off this one predicate, and a marker attached as context is not a link + /// in the source chain. + #[test] + fn an_unreachable_netd_is_recognized_through_the_contexts_stacked_on_it() { + let error = anyhow::Error::from(std::io::Error::from(std::io::ErrorKind::NotFound)) + .context(Unreachable) + .context("failed to connect to netd at /run/netd.sock") + .context("failed to prepare netd-managed networking"); + assert!(is_unreachable(&error)); + + let other = anyhow::anyhow!("netd remove_all failed: no such bridge") + .context("failed to prepare netd-managed networking"); + assert!(!is_unreachable(&other)); + } } diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index dd7a23d79..62c4bd78b 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -3,10 +3,10 @@ // SPDX-License-Identifier: Apache-2.0 use crate::app::{ - make_sys_config, resolved_networks, simulator_config_for_manifest, sync_tee_simulator_config, - Image, VmConfig, VmWorkDir, + make_sys_config, needs_netd_interface, resolved_networks, settle_vhost, + simulator_config_for_manifest, sync_tee_simulator_config, Image, VmConfig, VmWorkDir, }; -use crate::config::{Config, NetworkFilterMode, NetworkingMode}; +use crate::config::Config; use crate::main_service; use anyhow::{Context, Result}; use fs_err as fs; @@ -279,18 +279,22 @@ Compose file content (first 200 chars): gateway_enabled: app_compose.gateway_enabled(), }; - if !dry_run - && config.cvm.network_filter.mode == NetworkFilterMode::Libvirt - && resolved_networks(&manifest, &config.cvm) - .iter() - .any(|network| network.mode == NetworkingMode::Bridge) - { + let mut runtime_networks = resolved_networks(&manifest, &config.cvm); + // Settle the data plane before anything reads it, so `vhost_enabled()` is a + // decision rather than a request and the launch does not warn about a + // `/dev/vhost-net` the netdev it then builds never opens. + settle_vhost(&mut runtime_networks); + // Bridge and macvtap host interfaces belong to netd, whose lifecycle + // one-shot does not manage. Refusing is the honest answer: the alternative + // was to quietly build a different interface here than the server would, + // and then report the VM as if it had the one it asked for. + if !dry_run && runtime_networks.iter().any(needs_netd_interface) { anyhow::bail!( - "one-shot execution does not manage libvirt-filtered TAP lifecycle; run the VMM server directly or use --dry-run" + "one-shot execution does not manage netd interface lifecycle, which bridge and \ + macvtap networking need; run the VMM server directly or use --dry-run" ); } - let runtime_networks = resolved_networks(&manifest, &config.cvm); let process_configs = vm_builder_config .config_qemu(&workdir_path, &config.cvm, &gpus, &runtime_networks) .context("Failed to build QEMU configuration")?; diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 1a533434f..f4e1a7b52 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -321,17 +321,30 @@ def encrypt_env(envs, hex_public_key: str) -> str: def parse_port_mapping(port_str: str) -> Dict: - """Parse a port mapping string into a dictionary.""" + """Parse a port mapping string into a dictionary. + + Accepts an optional "@" suffix naming which NIC the traffic enters + through. Without it the VMM picks: the first user-mode NIC, else the first + bridge NIC. A single-NIC VM never needs it. + """ + nic_index = None + if "@" in port_str: + port_str, _, nic = port_str.rpartition("@") + # `int()` alone would take "1_0" as 10, " 1" as 1, and "+1" as 1. A NIC + # index is a position in a list the user wrote, so only digits are it. + if not (nic.isascii() and nic.isdigit()): + raise argparse.ArgumentTypeError(f"Invalid NIC index: {nic}") + nic_index = int(nic) parts = port_str.split(":") if len(parts) == 3: - return { + mapping = { "protocol": parts[0], "host_address": "127.0.0.1", "host_port": int(parts[1]), "vm_port": int(parts[2]), } elif len(parts) == 4: - return { + mapping = { "protocol": parts[0], "host_address": parts[1], "host_port": int(parts[2]), @@ -339,6 +352,9 @@ def parse_port_mapping(port_str: str) -> Dict: } else: raise argparse.ArgumentTypeError(f"Invalid port mapping format: {port_str}") + if nic_index is not None: + mapping["nic_index"] = nic_index + return mapping def read_utf8(filepath: str) -> str: @@ -919,8 +935,18 @@ def create_vm(self, args) -> None: params["kms_urls"] = args.kms_url if args.gateway_url: params["gateway_urls"] = args.gateway_url - if args.net: - params["networking"] = {"mode": args.net} + # "auto" is what a fresh deployment already does, so it only means + # something to `update`, where it clears a pinned count. + net_queues = None if args.net_queues == "auto" else args.net_queues + if args.net or args.net_vhost is not None or net_queues: + networking = {} + if args.net: + networking["mode"] = args.net + if args.net_vhost is not None: + networking["vhost"] = args.net_vhost + if net_queues: + networking["queues"] = net_queues + params["networking"] = networking app_id = args.app_id or self.calc_app_id(compose_content) print(f"App ID: {app_id}") @@ -1030,6 +1056,10 @@ def update_vm( no_gpus: bool = False, kms_urls: Optional[List[str]] = None, no_tee: Optional[bool] = None, + net: Optional[str] = None, + net_vhost: Optional[bool] = None, + net_vhost_inherit: bool = False, + net_queues: Optional[Union[int, str]] = None, ) -> None: """Update multiple aspects of a VM in one command.""" # Validate: --env-file requires --kms-url @@ -1153,6 +1183,75 @@ def update_vm( app_compose, indent=4, ensure_ascii=False ) + if net or net_vhost is not None or net_vhost_inherit or net_queues: + # The RPC replaces the whole NIC list, so merge into what the VM + # already has rather than silently dropping its other interfaces or + # un-pinning a bridge it was deployed with. + if vm_info_response is None: + vm_info_response = self.rpc_call("GetInfo", {"id": vm_id}) + if not vm_info_response.get("found", False): + raise Exception(f"VM with ID {vm_id} not found") + configuration = vm_info_response["info"].get("configuration") or {} + current = configuration.get("networks") or [] + if not current and configuration.get("networking"): + current = [configuration["networking"]] + if len(current) > 1: + raise Exception( + "this VM has multiple network interfaces; edit them through the " + "web UI or the UpgradeApp API rather than these flags" + ) + # Only the fields the deployment RPC accepts back travel with the + # update. macvtap_mode is node-controlled and can never be changed, + # so resending it can only fail if the node changed meanwhile. An + # empty mode is meaningful: it says the VM never named a backend + # and still follows the node's. + source = current[0] if current else {} + networking = { + key: source[key] + for key in ("mode", "bridge_name", "parent", "vhost", "queues") + if source.get(key) not in (None, "") + } + if net == "default": + # The only way back to "whatever backend the node runs". Without + # it a VM that named a mode once is pinned to it for life, since + # the merge above carries the reported mode forward on every + # later update. The data plane keeps whatever it was told. + networking.pop("mode", None) + networking.pop("bridge_name", None) + networking.pop("parent", None) + elif net: + networking["mode"] = net + # A field belongs to the mode that owns it. Carrying a bridge into + # a macvtap request, or a parent into a bridge one, asks the server + # about a field the caller never typed and has no flag to clear. + mode = networking.get("mode", "") + if mode and mode != "bridge": + networking.pop("bridge_name", None) + if mode and mode != "macvtap": + networking.pop("parent", None) + # Same rule for the data plane. User networking has neither a vhost + # backend nor multiple queues, so carrying an inherited pin into it + # is rejected for a flag the operator never typed -- and the two + # flags that would clear it are the ones they have not found yet. + # An explicitly typed value still earns the error: that one is + # theirs to be wrong about. + if mode == "user": + if net_vhost is None: + networking.pop("vhost", None) + if not net_queues: + networking.pop("queues", None) + if net_vhost is not None: + networking["vhost"] = net_vhost + elif net_vhost_inherit: + networking.pop("vhost", None) + if net_queues == "auto": + networking.pop("queues", None) + elif net_queues: + networking["queues"] = net_queues + upgrade_params["update_networking"] = True + upgrade_params["networks"] = [networking] + updates.append(f"networking ({networking})") + if user_config: upgrade_params["user_config"] = user_config updates.append("user config") @@ -1247,6 +1346,38 @@ def show_info(self, vm_id: str, json_output: bool = False) -> None: if info.get("shutdown_progress"): print(f"Shutdown: {info['shutdown_progress']}") + interfaces = info.get("interfaces") or [] + if interfaces: + print("\nNetwork Interfaces:") + for iface in interfaces: + parts = [ + f"{iface.get('netdev_id') or '-':<6}", + f"{iface.get('mode') or '-'}/{iface.get('backend') or '-'}", + iface.get("mac") or "-", + ] + if iface.get("bridge_name"): + parts.append(f"bridge={iface['bridge_name']}") + if iface.get("macvtap_mode"): + parts.append(f"macvtap_mode={iface['macvtap_mode']}") + # Absent, not false: custom mode carries an operator-written + # netdev string the VMM never parses, so it reports no data + # plane rather than asserting the resolved default over one that + # may well say vhost=on,queues=8. + vhost = iface.get("vhost") + parts.append( + "vhost=" + ("-" if vhost is None else ("on" if vhost else "off")) + ) + parts.append(f"queues={iface.get('queues') or '-'}") + print(" " + " ".join(parts)) + # A stopped VM has no interfaces to describe, so these are what its + # next launch would build -- which can differ from its last one. + # + # The server's own predicate, not the status string: a VM being + # removed with QEMU still up reports the interfaces that process + # built, and no status value says so. + if not info.get("running", False): + print(" (not running; shown as its next launch would build them)") + events = info.get("events", []) if events: print("\nRecent Events:") @@ -1536,6 +1667,26 @@ def save_whitelist(whitelist: List[str]) -> None: json.dump({"trusted_signers": whitelist}, f, indent=2) +def queue_count(value: str) -> Union[int, str]: + """Parse a queue pair count the node could act on, or "auto" to stop pinning one. + + Zero would otherwise reach the wire as "unset" and be answered with the + default, and a negative one as a decoding error naming a column offset -- + neither of which tells the caller what they asked for was impossible. + """ + if value == "auto": + return "auto" + try: + count = int(value) + except ValueError: + raise argparse.ArgumentTypeError(f"'{value}' is not a whole number or 'auto'") + if count < 1: + raise argparse.ArgumentTypeError( + f"queue pairs must be at least 1, or 'auto' to follow the vCPU count; got {count}" + ) + return count + + def main(): """Parse arguments and dispatch to the appropriate command handler.""" parser = argparse.ArgumentParser(description="dstack-vmm CLI - Manage VMs") @@ -1772,7 +1923,7 @@ def _patched_format_help(): "--port", action="append", type=str, - help="Port mapping in format: protocol[:address]:from:to", + help="Port mapping in format: protocol[:address]:from:to[@nic]", ) deploy_parser.add_argument( "--gpu", @@ -1831,9 +1982,31 @@ def _patched_format_help(): ) deploy_parser.add_argument( "--net", - choices=["bridge", "user"], + choices=["bridge", "user", "macvtap"], help="Networking mode (default: use global config)", ) + net_vhost = deploy_parser.add_mutually_exclusive_group() + net_vhost.add_argument( + "--net-vhost", + dest="net_vhost", + action="store_true", + default=None, + help="Use the host kernel vhost-net data plane (default: use global config)", + ) + net_vhost.add_argument( + "--net-no-vhost", + dest="net_vhost", + action="store_false", + help="Keep packet processing in the QEMU main loop", + ) + deploy_parser.add_argument( + "--net-queues", + type=queue_count, + metavar="N", + help="virtio-net queue pairs, bounded by the node's max_net_queues. " + "Without --net, the node's own networking mode is kept " + "(default: use global config)", + ) # Images command lsimage_parser = subparsers.add_parser("lsimage", help="List available images") @@ -1906,7 +2079,7 @@ def _patched_format_help(): action="append", type=str, required=True, - help="Port mapping in format: protocol[:address]:from:to (can be used multiple times)", + help="Port mapping in format: protocol[:address]:from:to[@nic] (can be used multiple times)", ) # Update (all-in-one) command @@ -1934,13 +2107,49 @@ def _patched_format_help(): "--env-file", help="File with environment variables to encrypt" ) update_parser.add_argument("--user-config", help="Path to user config file") + update_parser.add_argument( + "--net", + choices=["bridge", "user", "macvtap", "default"], + help=( + "Networking mode (applies from the next boot). 'default' stops " + "pinning a mode and follows the node's, the way --net-queues auto " + "and --net-vhost-default stop pinning the data plane" + ), + ) + update_net_vhost = update_parser.add_mutually_exclusive_group() + update_net_vhost.add_argument( + "--net-vhost", + dest="net_vhost", + action="store_true", + default=None, + help="Use the host kernel vhost-net data plane", + ) + update_net_vhost.add_argument( + "--net-no-vhost", + dest="net_vhost", + action="store_false", + help="Keep packet processing in the QEMU main loop", + ) + update_net_vhost.add_argument( + "--net-vhost-default", + dest="net_vhost_inherit", + action="store_true", + help="Stop pinning vhost and follow the node default again", + ) + update_parser.add_argument( + "--net-queues", + type=queue_count, + metavar="N", + help="virtio-net queue pairs, bounded by the node's max_net_queues. " + "Use 'auto' to stop pinning a count and follow the vCPU count again", + ) # Port mapping options (mutually exclusive with --no-ports) port_group = update_parser.add_mutually_exclusive_group() port_group.add_argument( "--port", action="append", type=str, - help="Port mapping in format: protocol[:address]:from:to (can be used multiple times)", + help="Port mapping in format: protocol[:address]:from:to[@nic] (can be used multiple times)", ) port_group.add_argument( "--no-ports", @@ -2077,6 +2286,10 @@ def _patched_format_help(): no_gpus=args.no_gpus if hasattr(args, "no_gpus") else False, kms_urls=args.kms_url, no_tee=args.no_tee, + net=args.net, + net_vhost=args.net_vhost, + net_vhost_inherit=getattr(args, "net_vhost_inherit", False), + net_queues=args.net_queues, ) elif args.command == "kms": if not args.kms_action: diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index 82653a62d..b07b36255 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -22,7 +22,44 @@ const CreateVmDialogComponent = { portMappingEnabled: { type: Boolean, required: true }, networkingModes: { type: Array, required: true }, defaultBridge: { type: String, default: '' }, + maxNetQueues: { type: Number, default: 0 }, defaultNetworkingLabel: { type: String, required: true }, + defaultModeTunable: { type: Boolean, default: false }, + defaultVhostOn: { type: Boolean, default: false }, + }, + methods: { + // Whether this NIC will end up on the vhost data plane. An unset select + // means it follows the node, and a node with vhost off gives one queue pair + // however many vCPUs the VM has -- so the answer is not readable from this + // row alone. + vhostOn(network: { vhost?: string }) { + if (network.vhost === 'on') { + return true; + } + if (network.vhost === 'off') { + return false; + } + return (this as any).defaultVhostOn; + }, + // What an empty queues field actually resolves to. It is the vCPU count + // only when vhost is on: with vhost off the backend has no multiqueue data + // plane and the NIC gets exactly one queue pair. + queuesHint(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'virtio-net queue pairs. Empty means one queue pair, because vhost is off.'; + } + const cap = (this as any).maxNetQueues + ? `, capped at ${(this as any).maxNetQueues} on this node` + : ''; + return `virtio-net queue pairs. Empty follows the VM's vCPU count${cap}.`; + }, + queuesPlaceholder(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'queues: auto (1, vhost off)'; + } + const cap = (this as any).maxNetQueues ? ` (max ${(this as any).maxNetQueues})` : ''; + return `queues: auto${cap}`; + }, }, emits: ['close', 'submit', 'load-compose'], template: /* html */ ` @@ -161,6 +198,7 @@ const CreateVmDialogComponent = {
{{ defaultNetworkingLabel }}
+ + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} Guest IP is assigned by host DHCP on that bridge and reported after boot. + + Leave empty to use the macvtap parent from vmm.toml. The forwarding mode stays node-controlled. + + + {{ defaultNetworkingLabel }} has no vhost-net or multiqueue data plane, so these two settings are recorded + but stay dormant until this node's default backend can carry them. +
- + @@ -223,7 +293,10 @@ const CreateVmDialogComponent = {
- +
diff --git a/dstack/vmm/ui/src/components/PortMappingEditor.ts b/dstack/vmm/ui/src/components/PortMappingEditor.ts index 0668bd952..70a4ad278 100644 --- a/dstack/vmm/ui/src/components/PortMappingEditor.ts +++ b/dstack/vmm/ui/src/components/PortMappingEditor.ts @@ -8,6 +8,10 @@ type PortEntry = { host_port: number | null; vm_port: number | null; custom_ip?: string; // User-entered IP for custom mode + // Which NIC the traffic enters through. Blank lets the VMM pick: the first + // user-mode NIC, else the first bridge NIC. A single-NIC VM never needs it, + // which is why the field only appears once a VM has more than one. + nic_index?: number | string | null; }; // ... keep your types as-is ... @@ -21,6 +25,9 @@ const PortMappingEditorComponent = { name: 'PortMappingEditor', props: { ports: { type: Array, required: true }, + // How many NICs the VM has. One NIC has nothing to choose between, so the + // column stays hidden rather than offering a pin that can only be 0. + nicCount: { type: Number, default: 1 }, }, // normalize on initial load @@ -63,6 +70,15 @@ const PortMappingEditorComponent = { + @@ -95,6 +111,7 @@ const PortMappingEditorComponent = { custom_ip: '', host_port: null, vm_port: null, + nic_index: null, }); }, diff --git a/dstack/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts index 5dc9568b1..4fa40e493 100644 --- a/dstack/vmm/ui/src/components/UpdateVmDialog.ts +++ b/dstack/vmm/ui/src/components/UpdateVmDialog.ts @@ -21,10 +21,47 @@ const UpdateVmDialogComponent = { portMappingEnabled: { type: Boolean, required: true }, networkingModes: { type: Array, required: true }, defaultBridge: { type: String, default: '' }, + maxNetQueues: { type: Number, default: 0 }, defaultNetworkingLabel: { type: String, required: true }, + defaultModeTunable: { type: Boolean, default: false }, + defaultVhostOn: { type: Boolean, default: false }, kmsEnabled: { type: Boolean, required: true }, composeHashPreview: { type: String, required: true }, }, + methods: { + // Whether this NIC will end up on the vhost data plane. An unset select + // means it follows the node, and a node with vhost off gives one queue pair + // however many vCPUs the VM has -- so the answer is not readable from this + // row alone. + vhostOn(network: { vhost?: string }) { + if (network.vhost === 'on') { + return true; + } + if (network.vhost === 'off') { + return false; + } + return (this as any).defaultVhostOn; + }, + // What an empty queues field actually resolves to. It is the vCPU count + // only when vhost is on: with vhost off the backend has no multiqueue data + // plane and the NIC gets exactly one queue pair. + queuesHint(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'virtio-net queue pairs. Empty means one queue pair, because vhost is off.'; + } + const cap = (this as any).maxNetQueues + ? `, capped at ${(this as any).maxNetQueues} on this node` + : ''; + return `virtio-net queue pairs. Empty follows the VM's vCPU count${cap}.`; + }, + queuesPlaceholder(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'queues: auto (1, vhost off)'; + } + const cap = (this as any).maxNetQueues ? ` (max ${(this as any).maxNetQueues})` : ''; + return `queues: auto${cap}`; + }, + }, emits: ['close', 'submit', 'load-compose'], template: /* html */ `
@@ -139,7 +176,10 @@ const UpdateVmDialogComponent = {
- +
@@ -150,24 +190,67 @@ const UpdateVmDialogComponent = {
{{ defaultNetworkingLabel }}
+ + + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} Guest IP is assigned by host DHCP on that bridge and reported after boot. + + Leave empty to use the macvtap parent from vmm.toml. The forwarding mode stays node-controlled. + + + {{ defaultNetworkingLabel }} has no vhost-net or multiqueue data plane, so these two settings are recorded + but stay dormant until this node's default backend can carry them. +
- +
diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index 1abb5ab75..12bbbf6de 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -92,6 +92,7 @@ type VmListItem = { shutdown_progress?: string; image_version?: string; interfaces?: VmmTypes.INetworkInterfaceStatus[]; + running?: boolean; configuration?: VmConfiguration; appCompose?: AppCompose; }; @@ -106,11 +107,29 @@ type PortFormEntry = { host_address?: string; host_port?: number | null; vm_port?: number | null; + /** + * Which NIC this mapping's traffic enters through. Unset lets the VMM pick. + * Carried through edits unchanged: `GetInfo` reports it and this form sends + * the whole list back, so dropping it here would silently unpin a mapping + * whenever anyone touched an unrelated field. + */ + // `v-model.number` leaves the raw string here when it does not parse, so + // an emptied box is `''` rather than `null`. See `normalizePorts`. + nic_index?: number | string | null; }; type NetworkFormEntry = { mode: string; bridge_name?: string; + /** Pinned at deployment for macvtap NICs; carried through edits unchanged. */ + parent?: string; + /** '' inherits the node default, otherwise 'on' or 'off'. */ + vhost?: string; + /** + * '' lets the queue count follow the vCPU count. A `number` once the operator + * types into the input: `v-model` casts for ``. + */ + queues?: string | number; }; type VmFormState = { @@ -351,6 +370,18 @@ fi return Array.from(new Set(fallback)); }); const defaultBridge = computed(() => config.value.networking?.default_bridge || ''); + const maxNetQueues = computed(() => config.value.networking?.max_queues || 0); + const defaultVhostOn = computed(() => !!config.value.networking?.default_vhost); + // Whether the node's own default backend can carry vhost-net and multiqueue. + // The RPC accepts the two tuning fields on a mode-less entry regardless -- + // deliberately, so a NIC that inherits its backend stays tunable across a node + // change -- but on a node whose default is user or custom they lie dormant, + // and an operator who sets them deserves to be told that rather than discover + // it in the interfaces panel afterwards. + const defaultModeTunable = computed(() => { + const mode = config.value.networking?.default_mode || ''; + return mode === 'bridge' || mode === 'macvtap'; + }); const defaultNetworkingLabel = computed(() => { const mode = config.value.networking?.default_mode || ''; if (mode === 'bridge') { @@ -413,6 +444,7 @@ fi host_address: port.host_address || '127.0.0.1', host_port: typeof port.host_port === 'number' ? port.host_port : null, vm_port: typeof port.vm_port === 'number' ? port.vm_port : null, + nic_index: typeof port.nic_index === 'number' ? port.nic_index : null, })); const normalizePorts = (ports: PortFormEntry[] = []): VmmTypes.IPortMapping[] => @@ -423,11 +455,23 @@ fi port.host_port === null || port.host_port === undefined ? Number.NaN : Number(port.host_port); const vmPort = port.vm_port === null || port.vm_port === undefined ? Number.NaN : Number(port.vm_port); + // An unpinned mapping must stay unpinned rather than become NIC 0: + // the VMM's own default is the first user-mode NIC, not the first NIC. + // `v-model.number` hands back the raw string when it does not parse, + // so a box the operator cleared arrives as `''`. `Number('')` is 0, + // which would pin to NIC 0 the mapping they just unpinned. + const nicIndex = + port.nic_index === null || port.nic_index === undefined || port.nic_index === '' + ? undefined + : Number(port.nic_index); return { protocol, host_address: (port.host_address || '127.0.0.1').trim() || '127.0.0.1', host_port: hostPort, vm_port: vmPort, + ...(Number.isInteger(nicIndex) && (nicIndex as number) >= 0 + ? { nic_index: nicIndex } + : {}), }; }) .filter( @@ -435,13 +479,7 @@ fi port.protocol.length > 0 && Number.isFinite(port.host_port) && Number.isFinite(port.vm_port), - ) - .map((port) => ({ - protocol: port.protocol, - host_address: port.host_address, - host_port: port.host_port, - vm_port: port.vm_port, - })); + ); const cloneNetworks = (configuration?: VmConfiguration | null): NetworkFormEntry[] => { const configured = configuration?.networks && configuration.networks.length > 0 @@ -450,16 +488,67 @@ fi return configured.map((network) => ({ mode: network.mode || '', bridge_name: network.bridge_name || '', + parent: network.parent || '', + vhost: network.vhost === null || network.vhost === undefined ? '' : (network.vhost ? 'on' : 'off'), + queues: network.queues ? String(network.queues) : '', })); }; + // Queue pairs, whatever shape the model is in. + // + // Not a string: Vue's `v-model` casts for ``, so this + // field is a `string` while it holds a value loaded from `GetInfo` and a + // `number` the moment the operator types into it. Assuming either one is how + // this threw a `TypeError` out of every deploy that set a queue count. + // + // The number input already refuses everything but a numeric literal, and the + // cast turns `2.7` into `2.7` rather than into `parseInt`'s `2`, so what is + // left to check is that the value is a whole number at least one. The node's + // cap is deliberately *not* checked here: the server widens it by whatever a + // VM already holds, so a client-side copy would refuse an update the server + // accepts and leave that VM's networking uneditable. + const parseQueueCount = (raw: unknown, label: string): number | undefined => { + if (raw === null || raw === undefined || raw === '') { + return undefined; + } + const queues = typeof raw === 'number' ? raw : Number(String(raw).trim()); + if (!Number.isInteger(queues) || queues < 1) { + throw new Error(`${label}: queue pairs must be a whole number of at least 1, or empty to follow the vCPU count; got '${raw}'`); + } + return queues; + }; + + // Nothing is filtered out. A mode-less entry is the "keep the node's backend, + // change only the data plane" override the RPC accepts, and is what a VM + // deployed that way reports back; dropping it would delete a NIC and renumber + // the ones after it, which changes their MAC addresses. + // A row added here starts with no mode, which the RPC reads as "keep the + // node's backend" rather than as a missing field, so the only way to reach an + // entry it refuses is to empty a loaded one's bridge or parent, and that + // earns an error rather than silence. const normalizeNetworks = (networks: NetworkFormEntry[] = []): VmmTypes.INetworkingConfig[] => networks - .map((network) => ({ - mode: (network.mode || '').trim(), - bridge_name: network.mode === 'bridge' ? (network.bridge_name || '').trim() : '', - })) - .filter((network) => network.mode.length > 0); + .map((network, index) => { + // Leave vhost and queues unset unless the operator picked something, so + // the node keeps owning them and can still change them later. + const entry: VmmTypes.INetworkingConfig = { + mode: (network.mode || '').trim(), + bridge_name: network.mode === 'bridge' ? (network.bridge_name || '').trim() : '', + parent: network.mode === 'macvtap' ? (network.parent || '').trim() : '', + }; + // The tuning controls are hidden for user mode, so sending values the + // operator cannot see would fail the deploy with nothing to fix. + if (network.mode !== 'user') { + if (network.vhost === 'on' || network.vhost === 'off') { + entry.vhost = network.vhost === 'on'; + } + const queues = parseQueueCount(network.queues, `network ${index + 1}`); + if (queues !== undefined) { + entry.queues = queues; + } + } + return entry; + }); function networkModeLabel(mode?: string | null) { if (!mode) { @@ -1014,6 +1103,11 @@ type CreateVmPayloadSource = { function showDeployDialog() { showCreateDialog.value = true; vmForm.value.encryptedEnvs = []; + // A cancelled deploy and "Clone config" both leave their networking behind + // in the shared form. Carrying it into the next deploy would silently pin + // that VM's backend, bridge, macvtap parent and data plane to another VM's + // -- invisibly, since the operator never opened the Networking section. + vmForm.value.networks = []; vmForm.value.app_id = null; vmForm.value.swapValue = 0; vmForm.value.swapUnit = 'GB'; @@ -1833,7 +1927,10 @@ type CreateVmPayloadSource = { config, networkingModes, defaultBridge, + maxNetQueues, defaultNetworkingLabel, + defaultModeTunable, + defaultVhostOn, composeHashPreview, updateComposeHashPreview, showDeployDialog, diff --git a/dstack/vmm/ui/src/styles/main.css b/dstack/vmm/ui/src/styles/main.css index 7a7b0ee72..113109a42 100644 --- a/dstack/vmm/ui/src/styles/main.css +++ b/dstack/vmm/ui/src/styles/main.css @@ -620,7 +620,7 @@ h1, h2, h3, h4, h5, h6 { .runtime-network-item { display: grid; - grid-template-columns: 1.1fr 0.6fr 0.8fr 1.4fr; + grid-template-columns: 1.1fr 0.6fr 0.8fr 1.4fr 0.7fr 0.7fr; gap: 12px; overflow-wrap: anywhere; } @@ -1420,13 +1420,35 @@ h1, h2, h3, h4, h5, h6 { font-size: 14px; } +/* Fixed track widths summed to a floor wider than the dialog, which put a + horizontal scrollbar under the Networking section on any window narrower than + about 930px. The tracks shrink and the tuning pair wraps instead. */ .network-config-row { display: grid; - grid-template-columns: 160px minmax(180px, 1fr) 100px; + grid-template-columns: minmax(120px, 160px) minmax(140px, 1fr) minmax(200px, auto) auto; gap: 12px; align-items: center; } +.network-config-tuning { + display: flex; + flex-wrap: wrap; + gap: 8px; + min-width: 0; +} + +.network-config-tuning select { + flex: 1 1 140px; + min-width: 110px; + max-width: 160px; +} + +.network-config-tuning input { + flex: 1 1 150px; + min-width: 90px; + max-width: 190px; +} + .network-config-placeholder { min-height: 1px; } diff --git a/dstack/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html index 139155032..965f7bcc2 100644 --- a/dstack/vmm/ui/src/templates/app.html +++ b/dstack/vmm/ui/src/templates/app.html @@ -80,7 +80,10 @@

dstack-vmm

:port-mapping-enabled="config.portMappingEnabled" :networking-modes="networkingModes" :default-bridge="defaultBridge" + :max-net-queues="maxNetQueues" :default-networking-label="defaultNetworkingLabel" + :default-mode-tunable="defaultModeTunable" + :default-vhost-on="defaultVhostOn" @close="showCreateDialog = false" @submit="createVm" @load-compose="loadComposeFile" @@ -95,7 +98,10 @@

dstack-vmm

:port-mapping-enabled="config.portMappingEnabled" :networking-modes="networkingModes" :default-bridge="defaultBridge" + :max-net-queues="maxNetQueues" :default-networking-label="defaultNetworkingLabel" + :default-mode-tunable="defaultModeTunable" + :default-vhost-on="defaultVhostOn" :kms-enabled="kmsEnabled(updateDialog.vm || {})" :compose-hash-preview="updateComposeHashPreview" @close="updateDialog.show = false" @@ -345,11 +351,16 @@

Port Mappings

VMM Network Interfaces

+ + Not running; shown as the next launch would build them. +
{{ networkModeLabel(iface.mode) }} / {{ iface.backend || '-' }} {{ iface.netdev_id || '-' }} - {{ iface.bridge_name || '-' }} + {{ iface.bridge_name || iface.macvtap_mode || '-' }} {{ iface.mac || '-' }} + vhost: {{ iface.vhost === null || iface.vhost === undefined ? '-' : (iface.vhost ? 'on' : 'off') }} + queues: {{ iface.queues || '-' }}
diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 26f221866..9ef0ecec4 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -46,7 +46,11 @@ max_allocable_memory_in_mb = 100_000 # MB qmp_socket = false # The user to run the VM as. If empty, the VM will be run as the current user. # Unique namespace when multiple dstack-vmm instances share one host. When -# empty, a stable value is derived from run_path. +# empty, a stable value is derived from run_path. It is what netd records on +# every host interface this VMM asks for, and the name space those interfaces' +# names are derived in. Two VMMs on one host must not share one: each would +# build interfaces at names the other can also produce. Left empty it is +# derived from run_path, which cannot collide. May not contain ":". instance_id = "" # Network choices that deployment RPC callers may make. Macvtap is excluded # by default because libvirt nwfilter applies only to bridge interfaces. @@ -54,6 +58,12 @@ allowed_network_modes = ["user", "bridge"] # Empty allowlists mean callers can only use the node networking defaults. allowed_bridges = [] allowed_macvtap_parents = [] +# Largest virtio-net queue pair count a deployment request may ask for. With +# vhost on, queue pairs otherwise default to the VM's vCPU count, capped at +# 16 (without vhost the default is a single queue pair); raising this +# above 16 widens what a caller may request without moving that default, and +# lowering it below 16 lowers the default too. +max_net_queues = 16 use_mrconfigid = true # QEMU flags @@ -111,6 +121,16 @@ product_name = "dstack" [cvm.networking] mode = "user" +# Kernel vhost-net data plane. Off by default: enabling it changes the +# virtio-net device of every bridge/macvtap VM on its next boot (vhost plus +# vCPU-scaled queue pairs) and requires /dev/vhost-net to be accessible to +# the account QEMU runs under — verify that first, or QEMU exits at launch. +# With it off, QEMU drains every packet on its single main loop thread, so a +# CVM cannot exceed one core's worth of packet processing no matter how many +# vCPUs it has. The user-mode backend has no vhost support and ignores this. +# Individual VMs may override it. +vhost = false + # for mode = "user" net = "10.0.2.0/24" dhcp_start = "10.0.2.10" @@ -120,21 +140,37 @@ restrict = false # bridge = "virbr0" # Optional filtering for bridge interfaces only. It does not apply to macvtap. -# "none" preserves the existing QEMU bridge-helper behavior and has no -# netd/libvirt dependency. +# "none" installs no nwfilter binding. It does not remove the netd dependency: +# netd builds every bridge and macvtap host interface whatever this says. [cvm.network_filter] mode = "none" filter = "clean-traffic" parameters = {} -# Shared privileged networking service. Only used when network_filter.mode is -# "libvirt". Socket filesystem permissions authorize clients. +# Shared privileged networking service. Required by bridge and macvtap +# networking: it builds every host interface those modes use, binds their +# nwfilters, and releases them again. It does not forward host ports. User mode +# and a caller-supplied netdev need nothing from it. Socket filesystem +# permissions authorize clients. [netd] socket = "/run/dstack/netd.sock" # Applied when netd creates the socket itself. A systemd socket unit controls # its own SocketMode instead. socket_mode = 0o660 libvirt_uri = "qemu:///system" +# The bridge filtering policy netd enforces and applies: whether a binding is +# required, which nwfilter it names, and with what parameters. netd holds this +# itself rather than taking it from each request, because a caller that chose +# the filter could name one that drops nothing and still satisfy a policy that +# only asked for "some filter". Left unset it follows [cvm.network_filter] in +# this same file, which is the whole answer when netd and the VMM share one +# vmm.toml. Set it explicitly when netd runs with a config that has no [cvm] +# section, so the daemon holding the privilege never infers policy from a file +# that does not state it. +#[netd.network_filter] +#mode = "libvirt" +#filter = "clean-traffic" +#parameters = {} [cvm.port_mapping] enabled = false