diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 0d16fd476..e3e7ed4d5 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -25,7 +25,7 @@ Individual VMs can override the global networking mode via: - **Web UI**: Networking dropdown in the deploy dialog - **API**: `networking: { mode: "bridge" }` in `VmConfiguration` -Only the mode is per-VM; the bridge interface name always comes from the global config. +The bridge interface name comes from the global config unless the node lists it in `cvm.allowed_bridges`. VMs may also override the vhost and queue settings — see [network-data-plane.md](network-data-plane.md). ## Host setup @@ -159,8 +159,9 @@ sudo chmod u+s /usr/lib/qemu/qemu-bridge-helper ## How it works -- VMM passes `-netdev bridge,id=net0,br=` to QEMU -- QEMU's bridge helper (setuid) creates a TAP device and attaches it to the bridge +- With more than one queue pair, or with libvirt filtering on, `netd` creates the TAP and the VMM passes `-netdev tap,id=net0,ifname=,...` — this is the usual case, since queue pairs default to the VM's vCPU count +- Otherwise the VMM passes `-netdev tap,id=net0,br=,helper=,vhost=on`, or `-netdev bridge,id=net0,br=` when vhost is off or no helper is found +- QEMU's bridge helper (setuid) creates a TAP device and attaches it to the bridge on the two helper paths - Guest MAC address is derived from SHA256 of the VM ID, with an optional configurable prefix (stable across restarts for DHCP IP consistency) - The host DHCP server (dnsmasq) assigns an IP to the VM - When QEMU exits, the TAP device is automatically destroyed @@ -201,6 +202,8 @@ Bridge and passt VMs can coexist. Set the global default in `vmm.toml` and overr vmm-cli.py deploy --name my-vm --image dstack-0.5.6 --compose app.yaml --net passt ``` -### vhost-net and TDX +### vhost-net and multiqueue -vhost-net (kernel data plane offload for virtio-net) is **not enabled** for bridge mode. TDX encrypts guest memory, which prevents the host kernel from performing DMA-based packet offload. The default QEMU userspace virtio backend is used instead. +Bridge NICs use the host kernel's vhost-net data plane by default, and can expose several virtio-net queue pairs. Both are configurable per node and per VM — see [network-data-plane.md](network-data-plane.md) for the knobs, the mode support matrix, and how to pick a queue count. + +vhost-net works in a TDX guest: the virtio rings and buffers live in shared, unencrypted memory so that a host-side backend can reach them, which is the same mechanism `vhost-vsock-pci` has always relied on. diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index 942146f0f..e81f7c8a8 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -162,10 +162,35 @@ sudo dstack-vmm --config ./vmm.toml \ --netd-socket /run/dstack-dev/netd.sock ``` -User networking and bridge networking with `mode = "none"` never connect to -`netd`. Libvirt mode fails closed if `netd` is unavailable. - -Filtered TAP netdevs currently set `vhost=off`. This keeps the initial backend -on the directly bound TAP path and avoids adding `/dev/vhost-net` permissions -to the QEMU user. It is a deliberate security-first throughput tradeoff; a -future configurable vhost mode requires equivalent filter integration tests. +User networking never connects to `netd`. Libvirt mode fails closed if `netd` +is unavailable. Bridge networking with `mode = "none"` connects only when it +needs more than one queue pair, as described below. + +Filtered TAP netdevs follow the node's `vhost` and `queues` settings like any +other TAP-backed NIC (see [network-data-plane.md](network-data-plane.md)). The +nwfilter binding is installed on the host TAP interface, so packets traverse it +whether they were written by QEMU or by a vhost worker; filtering is unaffected +by the data plane choice. Enabling vhost does require the QEMU user to be able +to open `/dev/vhost-net`. + +`netd` also creates the TAP for unfiltered bridge NICs that ask for more than +one queue pair, because `qemu-bridge-helper` returns a single descriptor and +cannot create a `multi_queue` device. Those TAPs carry no nwfilter binding, so +a multiqueue bridge node needs `netd` even when `network_filter.mode = "none"`. + +An empty filter name is what selects that unfiltered TAP, so `mode = "libvirt"` +with an empty `filter` is rejected at config load rather than quietly producing +an unbound TAP. + +Removal carries the same distinction: the VMM tells `netd` whether the interface +it is asking about was created with a binding, from a record made when it was +built rather than from configuration that may have changed since. A binding it +was told about must be gone before `netd` returns; otherwise `netd` still asks +libvirt to clear one — an interface name is reused by the same VM, and a +leftover binding's rules would be inherited — but a `libvirtd` it cannot reach +is a warning rather than a failure. So a node with `virsh` installed and no +running `libvirtd` can create and destroy multiqueue TAPs. The flag defaults to +true on the wire, so an older VMM's removals still drop their bindings. + +`netd` requires the `virsh` binary to be present whatever the filter mode; it is +`libvirtd` that unfiltered work does not need. diff --git a/docs/macvtap-networking.md b/docs/macvtap-networking.md index 544408d31..362de4e87 100644 --- a/docs/macvtap-networking.md +++ b/docs/macvtap-networking.md @@ -19,6 +19,9 @@ Configure a NIC through node configuration or an authorized VMM RPC request: `parent` must name an existing host interface. `macvtap_mode` may be `private`, `bridge`, `vepa`, or `passthru`; an empty value selects `private`. +Macvtap NICs also honour the `vhost` and `queues` settings described in +[network-data-plane.md](network-data-plane.md); netd creates the interface with +matching hardware queues and the launcher opens `/dev/tapN` once per queue. The configured netd socket permissions apply in the same way as for libvirt-filtered bridge networking. @@ -49,8 +52,9 @@ and the same deterministic MAC address passed to QEMU. Netd then: 4. reads its kernel-assigned ifindex and waits for `/dev/tap`; and 5. returns that runtime device path to the VMM. -The per-VM launcher opens the character device, places it at the fd referenced -by QEMU's `-netdev tap,fd=...` argument, and then execs QEMU. This keeps device +The per-VM launcher opens the character device once per queue pair, places the +descriptors at the fds referenced by QEMU's `-netdev tap,fd=...` (or `fds=...`) +argument, and then execs QEMU. This keeps device paths out of persistent VM configuration, works with both Supervisor and systemd process managers, and does not pass network fds through `sudo`. diff --git a/docs/network-data-plane.md b/docs/network-data-plane.md new file mode 100644 index 000000000..732822aeb --- /dev/null +++ b/docs/network-data-plane.md @@ -0,0 +1,220 @@ +# virtio-net data plane tuning + +Every CVM NIC has two knobs that decide how many packets it can move: whether +the host kernel's vhost-net data plane is used, and how many virtio-net queue +pairs the device exposes. vhost is set per node and overridable per VM; queue +pairs have no node-wide setting at all, for the reason given under +Configuration. + +## Why it matters + +Without vhost-net, QEMU drains every received packet on its single main-loop +thread. That thread is the ceiling, and it does not grow with vCPUs: + +``` +maximum packets per second ≈ 1 core ÷ per-packet main-loop cost +``` + +The per-packet cost varies with traffic shape — a few microseconds for uniform +synthetic streams, tens of microseconds for bidirectional short-connection +traffic — so the ceiling is a property of the workload, not a fixed number. +What is fixed is the shape of the failure: throughput climbs normally until the +main loop saturates at 100% of one core, then packets are dropped at the TAP +before they ever reach the guest. Guest-side counters stay clean, which makes +the cliff easy to misdiagnose as a network problem. + +Guest-side outbound traffic uses the same thread, so a busy guest pays the +cost twice over. + +`vhost=on` moves that work into the host kernel. That returns a whole core, but +it relocates the ceiling rather than removing +it: packets now arrive faster than a single guest receive queue can drain, and +the drops reappear at a higher rate. More queue pairs is what removes them, +which is why both are defaults: vhost everywhere, and a queue count that follows +the VM's vCPU count. The knob you are more likely to reach for is the other +direction — see [Choosing a queue count](#choosing-a-queue-count). + +## Configuration + +```toml +[cvm] +# Ceiling for both the default and what a deployment may request. +max_net_queues = 16 + +[cvm.networking] +mode = "bridge" +bridge = "dstack-br0" +vhost = true +``` + +Queue pairs are not a node setting. They default to the VM's vCPU count, capped +at 16, because the useful number follows the VM rather than the host — the guest +driver uses at most one queue pair per vCPU. A deployment overrides that per VM, +up to `max_net_queues`. + +Raising `max_net_queues` above 16 widens what a deployment may ask for without +moving the default's cap, so a larger VM never silently acquires a worse +default. Lowering it below 16 does lower the default too, because a node that +refuses a request for four queue pairs should not hand out sixteen by itself. +The hard ceiling from any source is 64. + +Turning vhost off also turns the multiqueue default off. Without vhost the QEMU +main loop drains every queue on one thread, so extra queues buy little while +still costing a netd interface, more MSI-X vectors, and a changed guest device. +An explicit queue count is still honoured without vhost, since that combination +is a deliberate request rather than a default. + +A VM overrides either value at deploy time, and `UpdateVm` changes them +afterwards — the new values apply from the VM's next boot: + +```bash +vmm-cli.py deploy --name my-vm --image dstack-0.5.9 --compose app.yaml \ + --net bridge --net-queues 4 +vmm-cli.py deploy --name latency-vm --image dstack-0.5.9 --compose app.yaml \ + --net bridge --net-no-vhost +``` + +The web UI exposes both per NIC in the deploy and update dialogs, alongside the +networking mode. Both fields are also on `NetworkingConfig` in the deployment +and update RPCs. A request that +sets only `vhost`/`queues` keeps the node's own networking mode, so tuning does +not force a caller to restate — or be allowed to choose — a backend. `queues` is +rejected above the node's `max_net_queues`; `vhost` is not otherwise restricted, +since it only affects the requesting VM. `GetMeta` reports +`networking.max_queues` so a client can present the real bound. + +The data plane settings are recorded only when a deployment asks for them. +Leave one out and it stays owned by the node, so changing `[cvm.networking]` +later — including setting `vhost = false` to roll the whole node back — still +reaches VMs deployed with some other networking override. + +Naming a backend is different: it pins that NIC's identity, resolved at +deployment. Its bridge or macvtap parent, its user-mode subnet and DHCP start, +and its MAC prefix are all fixed for the life of the VM, so a later edit to +those fields in `[cvm.networking]` does not reach it. A request that only tunes +pins nothing, including the backend it inherited. + +`GetInfo` reports that configuration back, and both `vmm-cli.py update` and the +web UI read it, change one field, and resend the rest. Two things follow. A +request may name a bridge or macvtap parent the node itself configured even when +the allowlists are empty: leaving the field out already yields exactly that +value, so echoing it grants nothing policy was withholding. And an update may +restate whatever its own VM already pinned, so that moving the node's default +out from under a VM does not leave that VM's configuration unsendable. A NIC +that inherited its backend reports an empty mode, which is the same thing it was +deployed with. + +Neither field reaches the CVM's measurement. The only measurement input the VMM +controls is `mr_config_id`, which covers the compose hash and instance info, so +retuning a NIC does not change app identity or require an on-chain update. + +## What each mode supports + +| Mode | netdev | vhost | queues > 1 | +|---|---|---|---| +| `user` | `user,...` | no backend | not supported | +| `bridge` | `tap,ifname=` via netd, else `tap,br=,helper=`, else `bridge,br=` | yes | yes, through netd | +| `bridge` with libvirt filtering | `tap,ifname=` | yes | yes, through netd | +| `macvtap` | `tap,fd=` / `tap,fds=` | yes | yes | +| `custom` | operator's own string | operator's own string | no, not settable | + +QEMU's `bridge` netdev accepts neither `vhost=` nor `queues=`, so enabling +vhost switches bridge mode to a `tap` netdev driven by the same setuid +`qemu-bridge-helper`. The VMM still needs no network privileges. The helper has +no compiled-in default path for the `tap` netdev, so the VMM probes the known +distribution locations; set `cvm.qemu_bridge_helper` if yours is elsewhere. If +no helper is found the NIC falls back to the non-vhost `bridge` netdev with a +warning, because vhost is a default and a default must not stop a node from +booting VMs. + +The helper returns exactly one descriptor, which is why more than one queue +pair in bridge mode is created by `netd` instead: it adds a persistent +`multi_queue` TAP that QEMU then opens once per queue. `netd` requires the +`virsh` binary to be installed even when nothing is filtered, though it does +not require a reachable `libvirtd`. That applies whether or +not libvirt filtering is on, so a bridge node needs `netd` to get the default +queue count (see [libvirt-network-filter.md](libvirt-network-filter.md)). +Without it, bridge NICs fall back to a single queue pair with a warning rather +than failing to launch; a VM that asked for a queue count explicitly still +fails, so the caller learns their request was not met. `netd` is probed by +connecting, not by looking for its socket file, because a `netd` that died +leaves the socket behind. One-shot `dstack-vmm run` has no netd lifecycle at +all and behaves like a node without it. `netd` reports back the +queue count it created, and the VMM refuses to launch on a mismatch — a `netd` +deployed separately as a root service can be older than the VMM asking it for +multiqueue, and QEMU would otherwise reject the interface from inside the +per-VM launcher. + +For macvtap, the per-VM launcher opens the `/dev/tapN` character device once +per queue pair and hands QEMU the descriptors as `fds=`. `netd` creates the +interface with matching `numtxqueues`/`numrxqueues`. + +Custom mode owns its whole netdev string, including any `vhost=`/`queues=` +options, and its guest device stays single-queue: the VMM cannot edit that +string, so it has no way to make a multiqueue device line agree with it. A +hand-written multiqueue netdev will not pair with a multiqueue guest device +today. + +Naming a backend that cannot carry vhost or a queue count, and then asking for +one, is refused — the request is yours to correct. Inheriting such a backend is +not, because the node chose it and may choose another tomorrow; the request +reads as off, or as one queue pair, until then. + +## Choosing a queue count + +The default suits bandwidth-bound workloads. Latency-sensitive ones should ask +for fewer: more queues spread receive processing over more vCPUs, and under TDX +a cross-vCPU wakeup costs an IPI and a VM exit. Measured on one 8-vCPU TDX CVM, +changing only the guest's channel count: + +| Queue pairs | Short-connection throughput | +|---|---| +| 1 | 22.3k conn/s | +| 2 | ~20k conn/s | +| 4 | 15–21k conn/s | +| 8 | 6.2–7.7k conn/s | + +The same CVM with 8 queues moved 3.0 Mpps of 64-byte UDP with no loss, against +roughly 600k with one queue. The trade is real in both directions, so a VM +serving many short connections should set `--net-queues 1` and measure. + +A VM with fewer vCPUs than queues leaves the extra pairs idle — `ethtool -l +eth0` reports the smaller number. An explicit over-provision is not rejected at +deployment, because `vmm-cli.py resize` can raise the vCPU count later. + +`vectors` is derived, never configured: `2N + 2`, one vector per queue +direction plus config and control. One queue pair emits no `mq=on` or +`vectors=` at all, leaving the guest device line byte for byte identical to the +one before this feature. The `-netdev` half does change wherever vhost is on, +since that is what selects the backend. + +## Requirements + +The account running QEMU must be able to open `/dev/vhost-net`, which is +`root:kvm 0660` on a stock host — add that account to the `kvm` group. The +`vhost_net` module autoloads on first open. + +`GetInfo` reports the data plane each interface actually got, so a bridge NIC +that fell back for want of a helper reads as `vhost: false` rather than +advertising something it is not using. For a VM that is not running there is no +interface to describe, so it reports what the next launch would build instead -- +the same calculation, against the node configuration and manifest as they stand +now, rather than the ones a finished boot ran under. + +If that account lacks access, QEMU exits at startup and the VM never boots. The +VMM does not pre-check this: QEMU need not share the VMM's credentials, so +refusing a launch on the VMM's own access would block deployments the host can +run. It only warns when the device node is missing outright, which is a fact +about the host rather than about either account. + +vhost-net works normally in a TDX guest: the virtio rings and buffers live in +shared, unencrypted memory precisely so a host-side backend can reach them. +This is the same mechanism behind `vhost-vsock-pci`, which dstack has always +used. + +On host kernels older than 6.4 the vhost worker is a free-standing kernel +thread: it is attached to the owner's cgroups, so `cpu.max` and cgroup +accounting do apply, but it is outside QEMU's thread group and so invisible to +`top -H` and to anything reading `/proc//task`. Since 6.4 it is a +`vhost_task` inside that thread group and shows up everywhere the VM's other +threads do. diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index c035e64d0..9cd386d53 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -52,6 +52,10 @@ message NetworkInterfaceStatus { optional string bridge_name = 4; // QEMU netdev id, e.g. "net0". optional string netdev_id = 5; + // Effective vhost-net data plane state for this interface. + bool vhost = 6; + // Effective virtio-net queue pairs. + uint32 queues = 7; } // Structured log or lifecycle event emitted by the guest or runtime. @@ -135,6 +139,12 @@ message NetworkingConfig { // Effective macvtap forwarding mode in responses. Deployment requests must // leave this empty because the mode is controlled by node configuration. string macvtap_mode = 4; + // Move packet processing into the host kernel vhost-net data plane. Unset + // inherits the node default. User mode has no vhost backend and ignores it. + optional bool vhost = 5; + // virtio-net queue pairs. Unset inherits the node default. Bounded by the + // node's cvm.max_net_queues. + optional uint32 queues = 6; } // Requested GPU layout for a CVM. @@ -301,6 +311,8 @@ message NetworkingCapabilities { reserved "forward_service_enabled"; // Default bridge configured in vmm.toml [cvm.networking].bridge. string default_bridge = 4; + // Largest virtio-net queue pair count a deployment request may ask for. + uint32 max_queues = 5; } // Aggregated metadata exposed through GetMeta. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 836176d4b..8f10ecfdb 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - config::{Config, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, Protocol}, + config::{Config, NetdInterface, Networking, NetworkingMode, ProcessAnnotation, Protocol}, logrotate, netd::{ self, InterfaceIdentity, PrepareBridgeRequest, PrepareMacvtapRequest, @@ -32,6 +32,7 @@ use rand::seq::SliceRandom; use serde::{Deserialize, Serialize}; use serde_json::json; use sha2::{Digest, Sha256}; +use std::cell::OnceCell; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use std::net::IpAddr; use std::path::{Path, PathBuf}; @@ -42,9 +43,15 @@ use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; pub(crate) use network::{ - resolve_networking, resolved_networks, validate_resolved_network, validate_resolved_networks, + clamp_queues_without_netd, filters_bridge_traffic, needs_netd_interface, netd_available, + netd_teardown, resolve_networking, resolved_networks, settle_vhost, validate_resolved_network, + validate_resolved_networks, }; pub use qemu::VmConfig; +// Exported so the RPC layer can assert that everything it reports is +// something it also accepts. +#[cfg(test)] +pub(crate) use vm_info::networking_to_proto; pub use workdir::VmWorkDir; mod host_share; @@ -355,7 +362,7 @@ impl App { let vm_id = manifest.id.clone(); let mut runtime_networks = vm_work_dir.runtime_networks(); if runtime_networks.is_empty() && cids_assigned.contains_key(&vm_id) { - runtime_networks = resolved_networks(&manifest, &self.config.cvm); + runtime_networks = self.inferred_runtime_networks(&manifest); if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); } @@ -454,7 +461,7 @@ impl App { append_boot_separator(&path); } - let mut runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); + let mut runtime_networks = self.runtime_networks(&vm_config.manifest); let devices = self.try_allocate_gpus(&vm_config.manifest)?; let gpu_host_config = self.config.cvm.gpu.clone(); let devices_to_sanitize = devices.clone(); @@ -547,19 +554,16 @@ impl App { vm: &VmConfig, networks: &mut [Networking], ) -> Result<()> { - if self.config.cvm.network_filter.mode == NetworkFilterMode::None - && !networks - .iter() - .any(|network| network.mode == NetworkingMode::Macvtap) + if !networks + .iter() + .any(|network| needs_netd_interface(network, &self.config.cvm)) { return Ok(()); } let qemu_uid = Uid::effective().as_raw(); let mut prepared = Vec::new(); for (nic_index, network) in networks.iter_mut().enumerate() { - if network.mode == NetworkingMode::Bridge - && self.config.cvm.network_filter.mode == NetworkFilterMode::None - { + if !needs_netd_interface(network, &self.config.cvm) { continue; } let identity = InterfaceIdentity { @@ -572,14 +576,27 @@ impl App { &network.mac_prefix_bytes(), nic_index, ); + let queues = network.queue_pairs(); + let filtered = filters_bridge_traffic(network, &self.config.cvm); let request = match network.mode { NetworkingMode::Bridge => NetdRequest::PrepareBridge(PrepareBridgeRequest { identity: identity.clone(), bridge: network.bridge.clone(), mac, qemu_uid, - filter: self.config.cvm.network_filter.filter.clone(), - parameters: self.config.cvm.network_filter.parameters.clone(), + // An unfiltered TAP is only created for multiqueue, where + // the node may not run libvirt at all. + filter: if filtered { + self.config.cvm.network_filter.filter.clone() + } else { + String::new() + }, + parameters: if filtered { + self.config.cvm.network_filter.parameters.clone() + } else { + BTreeMap::new() + }, + queues, }), NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap(PrepareMacvtapRequest { identity: identity.clone(), @@ -587,6 +604,7 @@ impl App { mac, qemu_uid, mode: network.macvtap_mode.clone(), + queues, }), NetworkingMode::User | NetworkingMode::Custom => continue, }; @@ -600,73 +618,207 @@ impl App { &self.config.netd.socket, &NetdRequest::Remove { identity: identity.clone(), + filtered, }, ) .await { warn!(%cleanup_error, "failed to roll back in-flight filtered network"); } - for identity in prepared.into_iter().rev() { - if let Err(cleanup_error) = netd::request( - &self.config.netd.socket, - &NetdRequest::Remove { identity }, - ) - .await - { - warn!(%cleanup_error, "failed to roll back prepared filtered network"); - } - } - return Err(error).context("failed to prepare libvirt-filtered networking"); + self.roll_back_prepared_networks(prepared).await; + let error = Err(error).context("failed to prepare netd-managed networking"); + // A netd deployed separately as a root service can be older + // than the VMM asking it for multiqueue, and it refuses the + // request before ever reaching the queue count -- an + // unfiltered TAP is spelled with an empty filter, which it + // does not accept either. Neither message mentions queues, + // so say it here. + return if queues > 1 { + error.with_context(|| { + format!( + "interface {nic_index} asked for {queues} queue pairs; a netd \ + deployed separately may predate multiqueue support" + ) + }) + } else { + error + }; } }; - if network.mode == NetworkingMode::Macvtap { - network.device = response - .device - .context("netd response omitted macvtap device")?; + prepared.push((identity.clone(), filtered)); + // netd built this one. Record it now, before anything else can + // fail, so teardown never has to re-derive it from a node + // configuration the operator may since have changed. + network.netd_interface = if filtered { + NetdInterface::Filtered + } else { + NetdInterface::Unfiltered + }; + // Everything below runs after netd already built a host interface, + // so a failure has to unwind the same way a failed Prepare does. + let accepted = (|| { + if network.mode == NetworkingMode::Macvtap { + network.device = response + .device + .clone() + .context("netd response omitted macvtap device")?; + } + // QEMU refuses a TAP whose IFF_MULTI_QUEUE state disagrees with + // its own `queues=`, and reports it from inside the per-VM + // launcher. netd echoes what it built, so a netd too old to + // understand the request fails here, where the reason is + // legible. + if queues > 1 && response.queues != Some(queues) { + bail!( + "netd prepared interface {nic_index} with {} queue pairs instead of \ + {queues}; its version may predate multiqueue support", + response.queues.map_or_else( + || "an unreported number of".to_string(), + |q| q.to_string() + ) + ); + } + Ok(()) + })(); + if let Err(error) = accepted { + self.roll_back_prepared_networks(prepared).await; + return Err(error); } - prepared.push(identity); } Ok(()) } + /// The NICs a VM has now, or would get if it were started. + /// + /// While QEMU is up this is what the launch actually built. Once it is + /// down the snapshot describes a boot that is over: the node configuration + /// and the VM's own manifest can both have changed since, so reporting it + /// would answer a question about the past with the grammar of the present. + /// Predict instead, the same way the next launch will -- including the + /// drop to a single queue pair on a node with no netd. + /// + /// `netd_reachable` is shared across a request rather than probed here: + /// the probe is a blocking connect that netd's serialized accept loop has + /// to service, and one status query covers many VMs. + fn effective_networks( + &self, + info: &vm_info::VmInfo, + netd_reachable: &OnceCell, + ) -> Vec { + if info.running && !info.runtime_networks.is_empty() { + return info.runtime_networks.clone(); + } + let available = *netd_reachable.get_or_init(|| netd_available(&self.config.netd.socket)); + self.merge_networks(&info.manifest, available).0 + } + + /// Launch-time view of a VM's NICs: node defaults merged in, the + /// vCPU-scaled queue count made concrete, and multiqueue dropped when this + /// node has no netd to build the interface. + pub(crate) fn runtime_networks(&self, manifest: &Manifest) -> Vec { + let available = netd_available(&self.config.netd.socket); + let (networks, clamped, vhost_denied) = self.merge_networks(manifest, available); + if clamped > 0 { + warn!( + id = %manifest.id, + "netd is not available, so {clamped} bridge interface(s) fall back to a single \ + queue pair; run dstack-vmm netd to let queue pairs scale with vCPUs" + ); + } + if vhost_denied > 0 { + warn!( + id = %manifest.id, + "no qemu-bridge-helper found, so {vhost_denied} bridge interface(s) fall back to \ + the non-vhost bridge netdev; set cvm.qemu_bridge_helper to enable vhost" + ); + } + networks + } + + /// A running VM whose snapshot is missing, because a VMM that predates the + /// snapshot -- or predates it recording what netd built -- started it. + /// + /// Guessing is all that is left, so guess the way that VMM would have, and + /// then write the guess down. Leaving the marker unset would make every + /// later teardown re-derive it from node configuration that may by then + /// have moved, which is the failure this snapshot exists to prevent. + fn inferred_runtime_networks(&self, manifest: &Manifest) -> Vec { + let mut networks = self.runtime_networks(manifest); + for network in &mut networks { + network.netd_interface = match netd_teardown(network, &self.config.cvm) { + Some(true) => NetdInterface::Filtered, + Some(false) => NetdInterface::Unfiltered, + None => NetdInterface::None, + }; + } + networks + } + + /// The merge itself, without the launch-time logging, plus how many NICs + /// lost multiqueue for want of netd. + fn merge_networks( + &self, + manifest: &Manifest, + netd_reachable: bool, + ) -> (Vec, usize, usize) { + let requested = if manifest.networks.is_empty() { + vec![self.config.cvm.networking.clone()] + } else { + manifest.networks.clone() + }; + let mut resolved = resolved_networks(manifest, &self.config.cvm); + let clamped = + clamp_queues_without_netd(&requested, &mut resolved, &self.config.cvm, netd_reachable); + let vhost_denied = settle_vhost(&mut resolved, &self.config.cvm); + (resolved, clamped, vhost_denied) + } + + /// Removes interfaces netd already built for a launch that then failed. + async fn roll_back_prepared_networks(&self, prepared: Vec<(InterfaceIdentity, bool)>) { + for (identity, filtered) in prepared.into_iter().rev() { + if let Err(cleanup_error) = netd::request( + &self.config.netd.socket, + &NetdRequest::Remove { identity, filtered }, + ) + .await + { + warn!(%cleanup_error, "failed to roll back prepared network interface"); + } + } + } + pub(crate) async fn remove_filtered_networks( &self, vm_id: &str, networks: &[Networking], ) -> Result<()> { - if self.config.cvm.network_filter.mode == NetworkFilterMode::None - && !networks - .iter() - .any(|network| network.mode == NetworkingMode::Macvtap) + if networks + .iter() + .all(|network| netd_teardown(network, &self.config.cvm).is_none()) { return Ok(()); } let mut first_error = None; for (nic_index, network) in networks.iter().enumerate().rev() { - if network.mode == NetworkingMode::Bridge - && self.config.cvm.network_filter.mode == NetworkFilterMode::None - { - continue; - } - if !matches!( - network.mode, - NetworkingMode::Bridge | NetworkingMode::Macvtap - ) { + let Some(filtered) = netd_teardown(network, &self.config.cvm) else { continue; - } + }; let identity = InterfaceIdentity { instance_id: self.config.cvm.instance_id.clone(), vm_id: vm_id.to_string(), nic_index, }; - if let Err(error) = - netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity }).await + if let Err(error) = netd::request( + &self.config.netd.socket, + &NetdRequest::Remove { identity, filtered }, + ) + .await { first_error.get_or_insert(error); } } if let Some(error) = first_error { - return Err(error).context("failed to remove libvirt-filtered networking"); + return Err(error).context("failed to remove netd-managed networking"); } Ok(()) } @@ -1059,7 +1211,7 @@ impl App { let already_running = cids_assigned.contains_key(&vm_id); let mut runtime_networks = vm_work_dir.runtime_networks(); if runtime_networks.is_empty() && already_running { - runtime_networks = resolved_networks(&manifest, &self.config.cvm); + runtime_networks = self.inferred_runtime_networks(&manifest); if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); } @@ -1160,11 +1312,15 @@ impl App { }); let total = infos.len() as u32; + // One probe for the whole page, and none at all when every VM is + // running and has its own snapshot to report. + let netd_reachable = OnceCell::new(); let vms = paginate(infos, request.page, request.page_size) .map(|vm| { let work_dir = self.work_dir(&vm.config.manifest.id)?; let info = vm.merged_info(vms.get(&vm.config.manifest.id), &work_dir); - Ok(info.to_pb(&self.config.gateway, &self.config.cvm, request.brief)) + let networks = self.effective_networks(&info, &netd_reachable); + Ok(info.to_pb(&self.config.gateway, request.brief, &networks)) }) .collect::>>()?; Ok(StatusResponse { @@ -1188,14 +1344,19 @@ impl App { pub async fn vm_info(&self, id: &str) -> Result> { let proc_state = self.supervisor.info(id).await?; - let state = self.lock(); - let Some(vm_state) = state.get(id) else { - return Ok(None); + // Snapshot under the lock, then release it: describing the VM can + // probe netd, and that is a blocking connect the global state lock has + // no business being held across. + let info = { + let state = self.lock(); + let Some(vm_state) = state.get(id) else { + return Ok(None); + }; + vm_state.merged_info(proc_state.as_ref(), &self.work_dir(id)?) }; - let info = vm_state - .merged_info(proc_state.as_ref(), &self.work_dir(id)?) - .to_pb(&self.config.gateway, &self.config.cvm, false); - Ok(Some(info)) + let netd_reachable = OnceCell::new(); + let networks = self.effective_networks(&info, &netd_reachable); + Ok(Some(info.to_pb(&self.config.gateway, false, &networks))) } pub(crate) fn vm_event_report(&self, cid: u32, event: &str, body: String) -> Result<()> { @@ -2241,6 +2402,10 @@ mod tests { dhcp_start: String::new(), restrict: false, netdev: String::new(), + vhost: None, + queues: None, + inherit_mode: false, + netd_interface: Default::default(), }]; workdir.put_manifest(&manifest)?; @@ -2506,6 +2671,10 @@ mod tests { dhcp_start: String::new(), restrict: false, netdev: String::new(), + vhost: None, + queues: None, + inherit_mode: false, + netd_interface: Default::default(), }]; let user_manifest = test_manifest(2048); let image = test_tdx_image(true); diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index 1b840154f..02f0c8339 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -10,11 +10,28 @@ use anyhow::{bail, Result}; use sha2::{Digest, Sha256}; use super::Manifest; -use crate::config::{CvmConfig, Networking, NetworkingMode}; +use crate::config::{ + CvmConfig, NetdInterface, NetworkFilterMode, Networking, NetworkingMode, MAX_NET_QUEUES, +}; -pub(crate) fn resolve_networking(networking: &Networking, cfg: &CvmConfig) -> Networking { +pub(crate) fn resolve_networking( + networking: &Networking, + cfg: &CvmConfig, + vcpu: u32, +) -> Networking { let mut resolved = cfg.networking.clone(); - resolved.mode = networking.mode; + // A deployment that only tuned the data plane never named a backend, so + // the node keeps deciding which one this NIC uses -- including after the + // operator changes it. + resolved.inherit_mode = networking.inherit_mode; + resolved.mode = if networking.inherit_mode { + cfg.networking.mode + } else { + networking.mode + }; + // Runtime state, never inherited from configuration or from a previous + // launch. Interface preparation sets it for the NICs it builds. + resolved.netd_interface = crate::config::NetdInterface::None; resolved.restrict = cfg.networking.restrict || networking.restrict; if !networking.bridge.is_empty() { resolved.bridge = networking.bridge.clone(); @@ -34,22 +51,207 @@ pub(crate) fn resolve_networking(networking: &Networking, cfg: &CvmConfig) -> Ne if !networking.netdev.is_empty() { resolved.netdev = networking.netdev.clone(); } + if networking.vhost.is_some() { + resolved.vhost = networking.vhost; + } + // Make the vCPU-scaled default concrete here, so every later stage -- + // netd preparation, the QEMU arguments, and removal after a VMM restart -- + // reads one number instead of recomputing it from a vCPU count it may no + // longer have. + resolved.queues = Some(match networking.queues { + // An explicit count is honoured whatever the data plane: multiqueue + // without vhost is a valid, if unusual, thing to ask for. + Some(queues) => queues, + // Without vhost the QEMU main loop drains every queue on one thread, + // so scaling up buys almost nothing while still costing a netd + // interface, extra vectors, and a changed device. Anyone turning vhost + // off is asking for the old data plane; give them the old shape too. + None if resolved.vhost_enabled() => { + Networking::default_queue_pairs(vcpu, cfg.max_net_queues) + } + None => 1, + }); resolved } pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec { - if manifest.networks.is_empty() { - vec![cfg.networking.clone()] + let node_default = [cfg.networking.clone()]; + let requested = if manifest.networks.is_empty() { + &node_default[..] } else { - manifest - .networks - .iter() - .map(|networking| resolve_networking(networking, cfg)) - .collect() + &manifest.networks[..] + }; + requested + .iter() + .map(|networking| resolve_networking(networking, cfg, manifest.vcpu)) + .collect() +} + +/// Whether netd must pre-create the host interface for this NIC. +/// +/// Macvtap always needs one. A bridge NIC needs one when libvirt filtering +/// binds an nwfilter to the TAP, and when multiqueue requires a persistent +/// `IFF_MULTI_QUEUE` device that `qemu-bridge-helper` cannot create. +pub(crate) fn needs_netd_interface(networking: &Networking, cfg: &CvmConfig) -> bool { + match networking.mode { + NetworkingMode::Macvtap => true, + NetworkingMode::Bridge => { + cfg.network_filter.mode == NetworkFilterMode::Libvirt || networking.queue_pairs() > 1 + } + NetworkingMode::User | NetworkingMode::Custom => false, + } +} + +/// Whether this NIC's host interface carries a libvirt nwfilter binding. +/// Macvtap never does, and a bridge NIC only does when the node filters. +pub(crate) fn filters_bridge_traffic(networking: &Networking, cfg: &CvmConfig) -> bool { + networking.mode == NetworkingMode::Bridge + && cfg.network_filter.mode == NetworkFilterMode::Libvirt +} + +/// Whether netd built this NIC's host interface, and if so whether it carries +/// an nwfilter binding. +/// +/// Interface preparation records this, because it is not derivable afterwards: +/// an operator can change `network_filter.mode` or `max_net_queues` while a VM +/// runs, and teardown has to undo what was built rather than what would be +/// built now. +pub(crate) fn netd_teardown(networking: &Networking, cfg: &CvmConfig) -> Option { + match networking.netd_interface { + NetdInterface::Filtered => Some(true), + NetdInterface::Unfiltered => Some(false), + // Either nothing was built, or this entry was persisted before + // preparation recorded the fact. Fall back to the derivation such an + // entry was created by; a Remove for an interface that does not exist + // is a no-op. + NetdInterface::None if needs_netd_interface(networking, cfg) => { + Some(filters_bridge_traffic(networking, cfg)) + } + NetdInterface::None => None, + } +} + +/// Drops a NIC back to one queue pair when multiqueue would need a netd +/// interface this node cannot provide. +/// +/// Queue pairs are a default now, not something the operator asked for, so a +/// node that has never deployed netd must keep launching bridge VMs. An +/// explicit per-VM request is left alone: the caller asked for it, and failing +/// at prepare tells them why far better than silently halving their throughput. +/// Returns how many NICs it dropped, so a launch can say so and a status +/// query, which runs the same calculation to describe a stopped VM, stays +/// silent. +pub(crate) fn clamp_queues_without_netd( + requested: &[Networking], + resolved: &mut [Networking], + cfg: &CvmConfig, + netd_available: bool, +) -> usize { + if netd_available { + return 0; + } + let mut clamped = 0; + for (networking, asked) in resolved.iter_mut().zip(requested) { + if networking.mode != NetworkingMode::Bridge + || asked.queues.is_some() + || !needs_netd_interface(networking, cfg) + // Filtering needs netd whatever the queue count, so dropping this + // NIC to one queue pair would not make it launchable. It would only + // describe it as something no launch can produce, and warn about a + // fallback that is not happening. + || filters_bridge_traffic(networking, cfg) + { + continue; + } + networking.queues = Some(1); + clamped += 1; + } + clamped +} + +/// Locations distributions install `qemu-bridge-helper` in. The helper is +/// setuid root and attaches an unprivileged TAP to a whitelisted bridge, which +/// is how bridge mode avoids giving the VMM `CAP_NET_ADMIN`. +const BRIDGE_HELPER_CANDIDATES: [&str; 3] = [ + "/usr/lib/qemu/qemu-bridge-helper", + "/usr/libexec/qemu-bridge-helper", + "/usr/local/libexec/qemu-bridge-helper", +]; + +/// Absolute path of `qemu-bridge-helper`, which QEMU's `tap` netdev, unlike its +/// `bridge` netdev, has no compiled-in default for. +/// +/// A configured path is passed through unchecked: the operator is naming a +/// binary for QEMU to exec, and QEMU need not see this filesystem. +pub(crate) fn find_bridge_helper<'a>( + configured: &'a str, + candidates: &[&'a str], +) -> Option<&'a str> { + let configured = configured.trim(); + if !configured.is_empty() { + return Some(configured); + } + candidates + .iter() + .copied() + .find(|candidate| Path::new(candidate).exists()) +} + +pub(crate) fn bridge_helper(cfg: &CvmConfig) -> Option<&str> { + find_bridge_helper(&cfg.qemu_bridge_helper, &BRIDGE_HELPER_CANDIDATES) +} + +/// Whether this NIC will actually run on the vhost-net data plane. +/// +/// A bridge NIC that neither needs a netd interface nor can find +/// `qemu-bridge-helper` falls back to QEMU's `bridge` netdev, which has no +/// vhost support. Both the QEMU arguments and the reported status read this, +/// so a VM is never described as using a data plane it did not get. +pub(crate) fn effective_vhost(networking: &Networking, cfg: &CvmConfig) -> bool { + if !networking.vhost_enabled() { + return false; } + networking.mode != NetworkingMode::Bridge + || needs_netd_interface(networking, cfg) + || bridge_helper(cfg).is_some() +} + +/// Makes the effective data plane concrete on a launch-time NIC list, and +/// returns how many interfaces asked for vhost and did not get it. +/// +/// `vhost` on a freshly resolved entry is still a *request*: `None` means +/// inherit, and a bridge NIC that cannot reach `qemu-bridge-helper` runs on the +/// non-vhost netdev whatever it asked for. Settling it once, here, is what lets +/// the QEMU arguments and the reported status read the same value -- and keeps +/// them reading it after the operator moves the helper out from under a VM that +/// is already running. +pub(crate) fn settle_vhost(networks: &mut [Networking], cfg: &CvmConfig) -> usize { + let mut denied = 0; + for networking in networks.iter_mut() { + let effective = effective_vhost(networking, cfg); + if networking.vhost_enabled() && !effective { + denied += 1; + } + networking.vhost = Some(effective); + } + denied +} + +/// Whether netd is reachable. A netd that died leaves its socket behind, so +/// existence alone would report a node as capable and fail every launch. +pub(crate) fn netd_available(socket: &Path) -> bool { + std::os::unix::net::UnixStream::connect(socket).is_ok() } pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { + // The vCPU-scaled default is bounded by construction; only an explicit + // request can exceed the hard cap. + if networking + .queues + .is_some_and(|queues| queues > MAX_NET_QUEUES) + { + bail!("networking queues must not exceed {MAX_NET_QUEUES}"); + } if networking.mode != NetworkingMode::Bridge { return Ok(()); } @@ -72,6 +274,34 @@ pub(crate) fn validate_resolved_networks(networks: &[Networking]) -> Result<()> Ok(()) } +/// Warns when a vhost NIC is about to launch on a host that has no +/// `/dev/vhost-net` at all. +/// +/// QEMU exits when `vhost=on` cannot open the device, and it does so from +/// inside the per-VM launcher where the reason is easy to miss. This puts the +/// remediation in the VMM log instead. +/// +/// Only existence is checked, and only as a warning. The device is +/// `root:kvm 0660`, so opening it — for reading as much as for writing — +/// answers a question about this process's credentials, and QEMU is not +/// necessarily this process: an externally started supervisor can run it under +/// another account. `stat` needs no permission on the device itself, so it +/// stays a statement about the host rather than about the VMM. +pub(crate) fn warn_if_vhost_net_missing(networks: &[Networking]) { + const VHOST_NET: &str = "/dev/vhost-net"; + if !networks.iter().any(Networking::vhost_enabled) { + return; + } + // The node is a kmod static device node, so it is present even before + // vhost_net is loaded; QEMU's open autoloads the module. + if !Path::new(VHOST_NET).exists() { + tracing::warn!( + "{VHOST_NET} is missing; vhost networking will fail to start. load the vhost_net \ + module, or set vhost = false in [cvm.networking]" + ); + } +} + /// Derives a deterministic, locally administered unicast MAC address. /// /// Index zero preserves the legacy single-NIC derivation. Later interfaces @@ -98,7 +328,325 @@ pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) #[cfg(test)] mod tests { - use super::mac_address_for_vm_index; + use super::{ + clamp_queues_without_netd, effective_vhost, mac_address_for_vm_index, needs_netd_interface, + netd_teardown, resolve_networking, resolved_networks, settle_vhost, + validate_resolved_networks, + }; + use crate::config::{Networking, NetworkingMode}; + + fn macvtap_network() -> Networking { + Networking { + mode: NetworkingMode::Macvtap, + bridge: String::new(), + parent: "eth0".into(), + macvtap_mode: String::new(), + device: String::new(), + mac_prefix: String::new(), + net: String::new(), + dhcp_start: String::new(), + restrict: false, + netdev: String::new(), + vhost: None, + queues: None, + inherit_mode: false, + netd_interface: Default::default(), + } + } + + fn node_config(mode: NetworkingMode) -> crate::config::CvmConfig { + use rocket::figment::providers::Format as _; + let config: crate::config::Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + let mut cvm = config.cvm; + cvm.networking.mode = mode; + cvm.networking.bridge = "br0".into(); + cvm.networking.parent = "eth0".into(); + cvm + } + + fn manifest_with(vcpu: u32, networks: Vec) -> crate::app::Manifest { + let mut manifest: crate::app::Manifest = serde_json::from_value(serde_json::json!({ + "id": "vm-1", "name": "n", "app_id": "a", "vcpu": vcpu, "memory": 2048, + "disk_size": 10, "image": "i", "port_map": [], "created_at_ms": 0, + })) + .unwrap(); + manifest.networks = networks; + manifest + } + + #[test] + fn queue_pairs_default_to_the_vcpu_count_up_to_the_cap() { + let cvm = node_config(NetworkingMode::Bridge); + for (vcpu, want) in [(1, 1), (2, 2), (8, 8), (16, 16), (32, 16), (128, 16)] { + let resolved = resolved_networks(&manifest_with(vcpu, vec![]), &cvm); + assert_eq!( + resolved[0].queue_pairs(), + want, + "vcpu {vcpu} should give {want} queue pairs" + ); + } + } + + #[test] + fn turning_vhost_off_also_turns_off_the_multiqueue_default() { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.networking.vhost = Some(false); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + assert!(!needs_netd_interface(&resolved[0], &cvm)); + + // Per-VM opt-out does the same thing. + let cvm = node_config(NetworkingMode::Bridge); + let mut asked = cvm.networking.clone(); + asked.vhost = Some(false); + let resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 1); + + // But an explicit queue count is still honoured without vhost. + let mut asked = cvm.networking.clone(); + asked.vhost = Some(false); + asked.queues = Some(4); + let resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 4); + } + + #[test] + fn lowering_the_request_ceiling_also_lowers_the_default() { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.max_net_queues = 2; + let resolved = resolved_networks(&manifest_with(16, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 2); + + // Raising it past the scaling cap widens requests, not the default. + cvm.max_net_queues = 32; + let resolved = resolved_networks(&manifest_with(24, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 16); + } + + #[test] + fn status_never_claims_a_data_plane_the_nic_did_not_get() { + let mut cvm = node_config(NetworkingMode::Bridge); + // No helper on this filesystem and no netd interface needed, so the + // NIC falls back to QEMU's `bridge` netdev, which has no vhost. + cvm.qemu_bridge_helper = String::new(); + let mut single = cvm.networking.clone(); + single.queues = Some(1); + let resolved = resolved_networks(&manifest_with(8, vec![single]), &cvm); + assert!(resolved[0].vhost_enabled()); + let fell_back = !effective_vhost(&resolved[0], &cvm); + assert_eq!(fell_back, super::bridge_helper(&cvm).is_none()); + + // A configured helper is taken at its word, so vhost is real. + cvm.qemu_bridge_helper = "/opt/qemu-bridge-helper".into(); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(effective_vhost(&resolved[0], &cvm)); + + // Multiqueue goes through netd, which needs no helper at all. + let mut mq = cvm.networking.clone(); + mq.queues = Some(4); + cvm.qemu_bridge_helper = String::new(); + let resolved = resolved_networks(&manifest_with(8, vec![mq]), &cvm); + assert!(needs_netd_interface(&resolved[0], &cvm)); + assert!(effective_vhost(&resolved[0], &cvm)); + } + + #[test] + fn an_explicit_queue_count_survives_resolution() { + let cvm = node_config(NetworkingMode::Bridge); + let mut asked = macvtap_network(); + asked.mode = NetworkingMode::Bridge; + asked.queues = Some(2); + let resolved = resolved_networks(&manifest_with(16, vec![asked]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 2); + } + + #[test] + fn user_mode_stays_single_queue_whatever_the_vcpu_count() { + let cvm = node_config(NetworkingMode::User); + let resolved = resolved_networks(&manifest_with(32, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 1); + } + + #[test] + fn without_netd_a_defaulted_bridge_drops_to_one_queue_but_a_request_does_not() { + let cvm = node_config(NetworkingMode::Bridge); + + // The default is ours to lower: a node that never deployed netd must + // keep launching bridge VMs. + let requested = vec![cvm.networking.clone()]; + let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 8); + clamp_queues_without_netd(&requested, &mut resolved, &cvm, false); + assert_eq!(resolved[0].queue_pairs(), 1); + assert!(!needs_netd_interface(&resolved[0], &cvm)); + + // An explicit request is left alone, so prepare fails where the caller + // can see why instead of silently halving their throughput. + let mut asked = cvm.networking.clone(); + asked.queues = Some(4); + let requested = vec![asked.clone()]; + let mut resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); + clamp_queues_without_netd(&requested, &mut resolved, &cvm, false); + assert_eq!(resolved[0].queue_pairs(), 4); + + // With netd present nothing is touched. + let requested = vec![cvm.networking.clone()]; + let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + clamp_queues_without_netd(&requested, &mut resolved, &cvm, true); + assert_eq!(resolved[0].queue_pairs(), 8); + } + + #[test] + fn validation_never_depends_on_this_process_reaching_vhost_net() { + // QEMU may run under different credentials, so a NIC that asks for + // vhost must validate on hosts where the VMM itself cannot open the + // device. Both of these hold whether or not /dev/vhost-net exists here. + let mut networking = macvtap_network(); + assert!(networking.vhost_enabled()); + validate_resolved_networks(&[networking.clone()]).unwrap(); + + networking.queues = Some(4); + validate_resolved_networks(&[networking]).unwrap(); + } + + #[test] + fn queue_counts_above_the_hard_bound_are_rejected() { + let mut networking = macvtap_network(); + networking.queues = Some(super::MAX_NET_QUEUES + 1); + let error = validate_resolved_networks(&[networking]).unwrap_err(); + assert!(error.to_string().contains("must not exceed")); + } + + /// The data plane a NIC actually gets is decided once, at launch, and + /// written into the runtime entry. Recomputing it later would let a report + /// about a running VM change under an operator's edit to node + /// configuration, describing a data plane QEMU is not using. + #[test] + fn settling_vhost_records_what_the_launch_decided() { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.qemu_bridge_helper = String::new(); + let mut single = cvm.networking.clone(); + single.queues = Some(1); + let manifest = manifest_with(8, vec![single]); + + // Whether this host has a helper is not the test's business; that it + // gets written down, once, is. + let helper_missing = super::bridge_helper(&cvm).is_none(); + let mut networks = resolved_networks(&manifest, &cvm); + assert!(networks[0].vhost_enabled(), "the request starts out on"); + assert_eq!( + settle_vhost(&mut networks, &cvm), + usize::from(helper_missing) + ); + assert_eq!(networks[0].vhost, Some(!helper_missing)); + // Settling an already-settled list reports nothing new, so a relaunch + // does not warn about a fallback that already happened. + assert_eq!(settle_vhost(&mut networks, &cvm), 0); + + // A configured helper is taken at its word, so the same NIC settles on. + cvm.qemu_bridge_helper = "/opt/qemu-bridge-helper".into(); + let mut with_helper = resolved_networks(&manifest, &cvm); + assert_eq!(settle_vhost(&mut with_helper, &cvm), 0); + assert_eq!(with_helper[0].vhost, Some(true)); + + // The entry the first launch settled keeps its answer: nothing about a + // running VM is recomputed from the configuration as it stands now. + assert_eq!(networks[0].vhost, Some(!helper_missing)); + } + + /// Dropping to a single queue pair is only worth doing when it makes the + /// NIC launchable. A filtered bridge needs netd whatever its queue count, + /// so clamping it would report a shape no launch can produce. + #[test] + fn a_filtered_bridge_is_not_clamped_because_it_cannot_help() { + use crate::config::NetworkFilterMode; + + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.network_filter.mode = NetworkFilterMode::Libvirt; + let requested = vec![cvm.networking.clone()]; + let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 8); + assert_eq!( + clamp_queues_without_netd(&requested, &mut resolved, &cvm, false), + 0 + ); + assert_eq!(resolved[0].queue_pairs(), 8); + + // Unfiltered, the same NIC does drop, because then it can launch. + let cvm = node_config(NetworkingMode::Bridge); + let mut resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert_eq!( + clamp_queues_without_netd(&requested, &mut resolved, &cvm, false), + 1 + ); + assert_eq!(resolved[0].queue_pairs(), 1); + } + + /// Teardown has to undo what was built. Node configuration is mutable and + /// a VM outlives an edit to it, so re-deriving "did netd build this?" at + /// removal time orphans TAPs and leaks nwfilter bindings whose ebtables + /// rules the next VM at the same deterministic interface name inherits. + #[test] + fn teardown_follows_what_was_built_not_what_configuration_now_says() { + use crate::config::{NetdInterface, NetworkFilterMode}; + + let filtering = { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.network_filter.mode = NetworkFilterMode::Libvirt; + cvm + }; + let unfiltered = node_config(NetworkingMode::Bridge); + + let mut built_filtered = filtering.networking.clone(); + built_filtered.queues = Some(1); + built_filtered.netd_interface = NetdInterface::Filtered; + // The operator turns filtering off while the VM runs. The binding is + // still there and still has to be deleted. + assert_eq!(netd_teardown(&built_filtered, &unfiltered), Some(true)); + + let mut built_unfiltered = unfiltered.networking.clone(); + built_unfiltered.queues = Some(4); + built_unfiltered.netd_interface = NetdInterface::Unfiltered; + // The operator turns filtering on. There is no binding to delete, and + // asking libvirt for one would fail the removal. + assert_eq!(netd_teardown(&built_unfiltered, &filtering), Some(false)); + + // A NIC netd never touched stays untouched, whatever the node now says. + let mut untouched = unfiltered.networking.clone(); + untouched.queues = Some(1); + assert_eq!(netd_teardown(&untouched, &unfiltered), None); + + // An entry persisted before preparation recorded the fact still gets + // torn down by the rule that created it. + let mut legacy = filtering.networking.clone(); + legacy.queues = Some(1); + assert_eq!(legacy.netd_interface, NetdInterface::None); + assert_eq!(netd_teardown(&legacy, &filtering), Some(true)); + } + + /// Resolution produces launch input, never a claim about what exists. + #[test] + fn resolution_never_carries_a_stale_interface_record() { + use crate::config::NetdInterface; + + let cvm = node_config(NetworkingMode::Bridge); + // Single queue and no filtering, so nothing but a stale record could + // make teardown believe netd built something. + let mut previous = cvm.networking.clone(); + previous.queues = Some(1); + previous.netd_interface = NetdInterface::Filtered; + assert_eq!(netd_teardown(&previous, &cvm), Some(true)); + + let resolved = resolve_networking(&previous, &cvm, 4); + assert_eq!(resolved.netd_interface, NetdInterface::None); + assert_eq!(netd_teardown(&resolved, &cvm), None); + } #[test] fn primary_mac_keeps_legacy_derivation_and_later_nics_are_distinct() { diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index db2fd39d4..b4da1302d 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -9,14 +9,15 @@ use super::{ hugepage_numa_nodes, image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, - network::{mac_address_for_vm_index, validate_resolved_networks}, + network::{ + bridge_helper, mac_address_for_vm_index, needs_netd_interface, validate_resolved_networks, + warn_if_vhost_net_missing, + }, pci_numa_node, round_up, GpuConfig, VmWorkDir, }; use crate::{ app::Manifest, - config::{ - CvmConfig, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, - }, + config::{CvmConfig, CvmPlatform, Networking, NetworkingMode, ProcessAnnotation}, netd::{tap_name, InterfaceIdentity}, vm_launcher::{ChildCommand, LaunchSpec, OpenFile}, }; @@ -167,6 +168,14 @@ fn create_hd( Ok(()) } +fn on_off(enabled: bool) -> &'static str { + if enabled { + "on" + } else { + "off" + } +} + fn virtio_pci_device(device: &str, snp: bool) -> String { if snp { format!("{device},disable-legacy=on,iommu_platform=true") @@ -179,6 +188,34 @@ struct PreparedVolume { source: String, } +/// First descriptor the per-VM launcher may hand to QEMU. Zero through two are +/// the standard streams. +const FIRST_INHERITED_FD: i32 = 3; + +/// Descriptors the launcher opens for each macvtap NIC, one per queue pair. +/// +/// Both the launcher's open list and the `-netdev` arguments derive from this +/// one layout, so they cannot disagree about which descriptor belongs to which +/// NIC. +fn macvtap_fd_layout(networks: &[Networking]) -> Vec> { + let mut next_fd = FIRST_INHERITED_FD; + networks + .iter() + .map(|network| { + if network.mode != NetworkingMode::Macvtap { + return Vec::new(); + } + (0..network.queue_pairs()) + .map(|_| { + let fd = next_fd; + next_fd += 1; + fd + }) + .collect() + }) + .collect() +} + struct PreparedQemuLaunch { workdir: VmWorkDir, platform: CvmPlatform, @@ -209,6 +246,7 @@ impl PreparedQemuLaunch { let platform = cfg.resolved_platform(); let networks = networks.to_vec(); validate_resolved_networks(&networks)?; + warn_if_vhost_net_missing(&networks); let volumes = vm .manifest .volumes @@ -396,14 +434,17 @@ impl VmConfig { swtpm: Option, swtpm_socket: Option, ) -> Result> { + // Each queue pair is a separate open of the same macvtap character + // device; the kernel attaches one tap queue per open. let open_files = prepared .networks .iter() - .enumerate() - .filter(|(_, network)| network.mode == NetworkingMode::Macvtap) - .map(|(index, network)| OpenFile { - fd: (3 + index) as i32, - path: network.device.clone().into(), + .zip(macvtap_fd_layout(&prepared.networks)) + .flat_map(|(network, fds)| { + fds.into_iter().map(|fd| OpenFile { + fd, + path: network.device.clone().into(), + }) }) .collect(); let spec = LaunchSpec { @@ -589,6 +630,7 @@ impl QemuCommandBuilder<'_> { } fn configure_networking(&self, command: &mut Command) -> Result<()> { + let macvtap_fds = macvtap_fd_layout(&self.prepared.networks); let hostfwd_index = self .prepared .networks @@ -601,12 +643,20 @@ impl QemuCommandBuilder<'_> { &networking.mac_prefix_bytes(), index, ); - let net_device = virtio_pci_device( - &format!("virtio-net-pci,netdev={net_id},mac={mac}"), - self.is_amd_sev_snp(), - ); + let queues = networking.queue_pairs(); + let vhost = networking.vhost_enabled(); + let mut device = format!("virtio-net-pci,netdev={net_id},mac={mac}"); + if queues > 1 { + // One vector per queue direction, plus config and control. + device.push_str(&format!(",mq=on,vectors={}", 2 * queues + 2)); + } + let net_device = virtio_pci_device(&device, self.is_amd_sev_snp()); let netdev = match networking.mode { NetworkingMode::User => { + // The user-mode backend has neither, so both are ignored + // here. A caller who *named* this mode and then asked for + // vhost or more than one queue pair is refused by the RPC; + // one who inherited it is not, and lands here. let mut netdev = format!( "user,id={net_id},net={},dhcpstart={},restrict={}", networking.net, @@ -627,24 +677,46 @@ impl QemuCommandBuilder<'_> { netdev } NetworkingMode::Bridge => { - tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge); - match self.cfg.network_filter.mode { - NetworkFilterMode::None => { - format!("bridge,id={net_id},br={}", networking.bridge) - } - NetworkFilterMode::Libvirt => { - let tap = tap_name(&InterfaceIdentity { - instance_id: self.cfg.instance_id.clone(), - vm_id: self.vm.manifest.id.clone(), - nic_index: index, - }); - // Keep the filtered backend conservative: QEMU - // uses the TAP path on which libvirt installed the - // nwfilter binding instead of opening vhost-net. - format!( - "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost=off" - ) + tracing::info!( + "bridge networking: mac={mac} bridge={} vhost={vhost} queues={queues}", + networking.bridge + ); + if needs_netd_interface(networking, self.cfg) { + // netd owns this TAP: libvirt filtering binds an + // nwfilter to it, and multiqueue needs the persistent + // IFF_MULTI_QUEUE device the bridge helper cannot make. + let tap = tap_name(&InterfaceIdentity { + instance_id: self.cfg.instance_id.clone(), + vm_id: self.vm.manifest.id.clone(), + nic_index: index, + }); + let mut netdev = format!( + "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost={}", + on_off(vhost) + ); + if queues > 1 { + netdev.push_str(&format!(",queues={queues}")); } + netdev + } else if let Some(helper) = vhost.then(|| bridge_helper(self.cfg)).flatten() { + // QEMU's `bridge` netdev has no vhost support, but the + // same setuid helper works behind a `tap` netdev, so + // the VMM still needs no network privileges. + format!( + "tap,id={net_id},br={},helper={helper},vhost=on", + networking.bridge + ) + } else if vhost { + // vhost defaults on, so a node whose helper sits + // somewhere unusual must keep booting VMs rather than + // lose every bridge NIC to a path lookup. + tracing::warn!( + "{net_id}: no qemu-bridge-helper found, falling back to the \ + non-vhost bridge netdev. set cvm.qemu_bridge_helper to enable vhost" + ); + format!("bridge,id={net_id},br={}", networking.bridge) + } else { + format!("bridge,id={net_id},br={}", networking.bridge) } } NetworkingMode::Custom => { @@ -659,7 +731,23 @@ impl QemuCommandBuilder<'_> { if networking.device.is_empty() { bail!("macvtap interface {index} has not been prepared by netd"); } - format!("tap,id={net_id},fd={},vhost=off", 3 + index) + let fds = macvtap_fds + .get(index) + .filter(|fds| !fds.is_empty()) + .with_context(|| { + format!("macvtap interface {index} has no launcher descriptors") + })?; + let selector = if fds.len() == 1 { + format!("fd={}", fds[0]) + } else { + let fds = fds + .iter() + .map(|fd| fd.to_string()) + .collect::>() + .join(":"); + format!("fds={fds}") + }; + format!("tap,id={net_id},{selector},vhost={}", on_off(vhost)) } }; command.arg("-netdev").arg(netdev); @@ -1021,8 +1109,8 @@ mod tests { }; use super::{ - amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, - PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, + amd_sev_snp_memory_backend_arg, macvtap_fd_layout, parse_amd_sev_snp_qmp_capabilities, + virtio_pci_device, PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, }; use crate::app::image::{Image, ImageInfo}; use crate::app::{needs_swtpm, GpuConfig, GpuSpec, Manifest, PortMapping, VmVolume, VmWorkDir}; @@ -1080,8 +1168,9 @@ mod tests { ); } - #[test] - fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { + /// Minimal launch fixture. Nothing it points at has to exist on disk; every + /// test overrides the fields it asserts on. + fn test_launch_fixture() -> (Config, VmConfig, PreparedQemuLaunch) { let mut config: Config = Figment::from(Toml::string(DEFAULT_CONFIG)) .extract() .unwrap(); @@ -1151,7 +1240,7 @@ mod tests { workdir: PathBuf::from("/does-not-exist/vm-1"), gateway_enabled: false, }; - let mut prepared = PreparedQemuLaunch { + let prepared = PreparedQemuLaunch { workdir: VmWorkDir::new("/does-not-exist/vm-1"), platform: CvmPlatform::Tdx, networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], @@ -1167,6 +1256,166 @@ mod tests { snp_host_data: None, snp_launch_params: None, }; + (config, vm, prepared) + } + + /// Builds the `-netdev`/`-device` pairs for one NIC layout. + fn net_args(config: &Config, networks: Vec) -> Vec { + let (_, vm, mut prepared) = test_launch_fixture(); + prepared.networks = networks; + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + process + .args + .windows(2) + .filter(|args| args[0] == "-netdev" || args[0] == "-device") + .map(|args| args[1].clone()) + .collect() + } + + fn bridge_network(config: &Config) -> Networking { + let mut networking = config.cvm.networking.clone(); + networking.mode = NetworkingMode::Bridge; + networking.bridge = "br0".into(); + networking + } + + #[test] + fn bridge_vhost_uses_the_bridge_helper_behind_a_tap_netdev() { + // QEMU's `bridge` netdev has no vhost support at all, so enabling the + // kernel data plane has to switch netdev types while keeping the same + // unprivileged setuid helper. + let (mut config, ..) = test_launch_fixture(); + config.cvm.qemu_bridge_helper = "/usr/lib/qemu/qemu-bridge-helper".into(); + let args = net_args(&config, vec![bridge_network(&config)]); + assert!(args.contains( + &"tap,id=net0,br=br0,helper=/usr/lib/qemu/qemu-bridge-helper,vhost=on".to_string() + )); + // A single queue pair must keep the historical device line byte for byte. + assert!(args.iter().any( + |arg| arg.starts_with("virtio-net-pci,netdev=net0,mac=") && !arg.contains("mq=on") + )); + } + + #[test] + fn a_missing_bridge_helper_is_reported_rather_than_guessed() { + // Configured paths are trusted verbatim: QEMU execs them, and it need + // not share this filesystem. + assert_eq!( + crate::app::network::find_bridge_helper(" /opt/qemu-bridge-helper ", &[]), + Some("/opt/qemu-bridge-helper") + ); + assert_eq!( + crate::app::network::find_bridge_helper("", &["/nonexistent/a", "/nonexistent/b"]), + None + ); + } + + #[test] + fn disabling_vhost_restores_the_legacy_bridge_netdev() { + let (config, ..) = test_launch_fixture(); + let mut networking = bridge_network(&config); + networking.vhost = Some(false); + let args = net_args(&config, vec![networking]); + assert!(args.contains(&"bridge,id=net0,br=br0".to_string())); + } + + #[test] + fn multiqueue_bridge_uses_the_netd_tap_and_derives_vectors() { + let (mut config, ..) = test_launch_fixture(); + config.cvm.instance_id = "vmm-a".into(); + let mut networking = bridge_network(&config); + networking.queues = Some(4); + let args = net_args(&config, vec![networking]); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=on,queues=4" + ))); + // vectors = 2 per queue pair, plus config and control. + assert!(args.iter().any(|arg| arg.contains("mq=on,vectors=10"))); + } + + #[test] + fn macvtap_queues_take_one_inherited_descriptor_each() { + let (config, ..) = test_launch_fixture(); + let mut first = config.cvm.networking.clone(); + first.mode = NetworkingMode::Macvtap; + first.parent = "eth0".into(); + first.device = "/dev/tap7".into(); + first.queues = Some(2); + let mut second = first.clone(); + second.device = "/dev/tap9".into(); + second.queues = Some(3); + + let networks = vec![first, second]; + let args = net_args(&config, networks.clone()); + assert!(args.contains(&"tap,id=net0,fds=3:4,vhost=on".to_string())); + assert!(args.contains(&"tap,id=net1,fds=5:6:7,vhost=on".to_string())); + + // The launcher must open exactly those descriptors, in that order. + let layout = macvtap_fd_layout(&networks); + assert_eq!(layout, vec![vec![3, 4], vec![5, 6, 7]]); + } + + #[test] + fn macvtap_keeps_a_single_fd_argument_for_one_queue() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.mode = NetworkingMode::Macvtap; + networking.parent = "eth0".into(); + networking.device = "/dev/tap7".into(); + let args = net_args(&config, vec![networking]); + assert!(args.contains(&"tap,id=net0,fd=3,vhost=on".to_string())); + } + + /// The operator owns a custom netdev string and the VMM cannot edit it, so + /// the generated device line must never claim more queues than that string + /// provides -- QEMU refuses the mismatch, from inside the per-VM launcher + /// where the reason is hard to see. + #[test] + fn custom_netdev_keeps_its_string_and_stays_single_queue() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.mode = NetworkingMode::Custom; + networking.netdev = "tap,id=net0,ifname=custom0,vhost=on,queues=8".into(); + // Even a queue count that reached the entry some other way is ignored. + networking.queues = Some(8); + let args = net_args(&config, vec![networking]); + assert!(args.contains(&"tap,id=net0,ifname=custom0,vhost=on,queues=8".to_string())); + assert!( + args.iter().all(|arg| !arg.contains("mq=on")), + "custom mode must not generate a multiqueue device line: {args:?}" + ); + } + + #[test] + fn user_mode_ignores_vhost_and_keeps_its_netdev() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.mode = NetworkingMode::User; + networking.vhost = Some(true); + let args = net_args(&config, vec![networking]); + assert!(args + .iter() + .any(|arg| arg.starts_with("user,id=net0,") && !arg.contains("vhost"))); + assert!(args.iter().any( + |arg| arg.starts_with("virtio-net-pci,netdev=net0,mac=") && !arg.contains("mq=on") + )); + } + + #[test] + fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { + let (mut config, vm, mut prepared) = test_launch_fixture(); let process = QemuCommandBuilder { vm: &vm, @@ -1234,6 +1483,7 @@ mod tests { for network in &mut prepared.networks { network.mode = NetworkingMode::Bridge; network.bridge = "br0".into(); + network.vhost = Some(false); } let process = QemuCommandBuilder { vm: &vm, @@ -1266,6 +1516,10 @@ mod tests { assert!(process.args.iter().any(|arg| { arg == &format!("tap,id=net0,ifname={expected_tap},script=no,downscript=no,vhost=off") })); + assert!(process + .args + .iter() + .all(|arg| !arg.contains("mq=on") && !arg.contains("vectors="))); prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock")); let process = QemuCommandBuilder { @@ -1304,6 +1558,10 @@ mod tests { dhcp_start: String::new(), restrict: false, netdev: "tap,id=wrong".into(), + vhost: None, + queues: None, + inherit_mode: false, + netd_interface: Default::default(), }]; let error = QemuCommandBuilder { vm: &vm, diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index eab200244..f70daff1b 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -11,16 +11,16 @@ use dstack_vmm_rpc as pb; use fs_err as fs; use supervisor_client::supervisor::ProcessInfo; -use super::{ - network::{mac_address_for_vm_index, resolved_networks}, - Manifest, VmState, VmWorkDir, -}; -use crate::config::{CvmConfig, GatewayConfig, Networking, NetworkingMode}; +use super::{network::mac_address_for_vm_index, Manifest, VmState, VmWorkDir}; +use crate::config::{GatewayConfig, Networking, NetworkingMode}; pub(crate) struct VmInfo { pub manifest: Manifest, pub workdir: PathBuf, pub status: &'static str, + /// Whether a QEMU process exists for this VM right now. The NICs it built + /// are real only while it does. + pub running: bool, pub uptime: String, pub exited_at: Option, pub instance_id: Option, @@ -51,16 +51,43 @@ fn networking_backend_name(mode: NetworkingMode) -> &'static str { } } -fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { +pub(crate) fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { + // An entry that inherited its backend reports no mode, so it must report + // none of the fields that only make sense alongside one: a mode-less + // override carrying, say, a parent is something the deployment RPC + // rejects, which would strand the VM's tuning as uneditable. + let pins_backend = !networking.inherit_mode; pb::NetworkingConfig { - mode: networking_mode_name(networking.mode).into(), - bridge_name: if networking.mode == NetworkingMode::Bridge { + // An entry that only tuned the data plane named no backend, and the + // deployment RPC spells that as an empty mode. Reporting the node's + // current mode here would turn a read-modify-write into a request to + // pin it -- which policy may not even permit the caller to make. + mode: if networking.inherit_mode { + String::new() + } else { + networking_mode_name(networking.mode).into() + }, + bridge_name: if pins_backend && networking.mode == NetworkingMode::Bridge { networking.bridge.clone() } else { String::new() }, - parent: networking.parent.clone(), - macvtap_mode: networking.macvtap_mode.clone(), + // Scope the macvtap fields to macvtap, the way bridge_name is scoped to + // bridge. Reporting an inherited parent on a bridge NIC produced a + // configuration that could be read but not sent back: the deployment + // RPC rejects `parent` outside macvtap mode. + parent: if pins_backend && networking.mode == NetworkingMode::Macvtap { + networking.parent.clone() + } else { + String::new() + }, + macvtap_mode: if pins_backend && networking.mode == NetworkingMode::Macvtap { + networking.macvtap_mode.clone() + } else { + String::new() + }, + vhost: networking.vhost, + queues: networking.queues, } } @@ -69,15 +96,20 @@ fn sanitize_optional>(value: Option) -> Option { } impl VmInfo { - pub fn effective_networks(&self, cvm: &CvmConfig) -> Vec { - if self.runtime_networks.is_empty() { - resolved_networks(&self.manifest, cvm) - } else { - self.runtime_networks.clone() - } - } - - pub fn to_pb(&self, gateway: &GatewayConfig, cvm: &CvmConfig, brief: bool) -> pb::VmInfo { + /// Takes no `CvmConfig` on purpose. Everything it reports about a VM's + /// data plane was decided when that VM launched and written into + /// `effective_networks`; consulting node configuration here is what let an + /// operator's edit change what a running VM was said to be using. + /// + /// `effective_networks` is passed in rather than derived for the same + /// reason, plus one more: a stopped VM's NICs are a prediction, and only + /// the caller can consult netd to make the prediction its launch would. + pub fn to_pb( + &self, + gateway: &GatewayConfig, + brief: bool, + effective_networks: &[Networking], + ) -> pb::VmInfo { let workdir = VmWorkDir::new(&self.workdir); let vm_config = workdir.manifest(); let custom_gateway_urls = vm_config @@ -91,8 +123,7 @@ impl VmInfo { .map(networking_to_proto) .collect::>(); let configured_networking = configured_networks.first().cloned(); - let interfaces = self - .effective_networks(cvm) + let interfaces = effective_networks .iter() .enumerate() .map(|(index, networking)| { @@ -108,6 +139,13 @@ impl VmInfo { bridge_name: (networking.mode == NetworkingMode::Bridge) .then(|| networking.bridge.clone()), netdev_id: Some(format!("net{index}")), + // Settled at launch, so an entry carrying no decision was + // written before this VMM recorded one -- by a build that + // had no vhost at all, which is what it should read as. + // Recomputing here instead would let an edit to node + // configuration change what a running VM is said to use. + vhost: networking.vhost.is_some() && networking.vhost_enabled(), + queues: networking.queue_pairs(), } }) .collect(); @@ -261,6 +299,7 @@ impl VmState { workdir: workdir.path().to_path_buf(), instance_id, status, + running: is_running, uptime, exited_at: Some(exited_at), boot_progress: self.state.boot_progress.clone(), @@ -276,7 +315,38 @@ impl VmState { #[cfg(test)] mod tests { - use super::sanitize_optional; + use super::{networking_to_proto, sanitize_optional}; + use crate::config::{Networking, NetworkingMode}; + + #[test] + fn a_reported_interface_can_be_sent_back_unchanged() { + // GetInfo output feeds UpdateVm, so anything it reports has to satisfy + // the deployment RPC's own validation. + let networking = Networking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + // Inherited from the node's [cvm.networking], not this NIC's own. + parent: "eth0".into(), + macvtap_mode: "private".into(), + device: String::new(), + mac_prefix: String::new(), + net: "10.0.2.0/24".into(), + dhcp_start: String::new(), + restrict: false, + netdev: String::new(), + vhost: Some(false), + queues: Some(2), + inherit_mode: false, + netd_interface: Default::default(), + }; + let proto = networking_to_proto(&networking); + assert_eq!(proto.mode, "bridge"); + assert_eq!(proto.bridge_name, "br0"); + assert!(proto.parent.is_empty()); + assert!(proto.macvtap_mode.is_empty()); + assert_eq!(proto.vhost, Some(false)); + assert_eq!(proto.queues, Some(2)); + } #[test] fn sanitize_optional_filters_empty_owned_values() { diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 2e82e67c6..3a01b0882 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -357,6 +357,10 @@ pub struct CvmConfig { pub qemu_pci_hole64_size: u64, /// QEMU hotplug_off pub qemu_hotplug_off: bool, + /// Path to `qemu-bridge-helper`, used to attach an unprivileged TAP to a + /// host bridge. Empty probes the known distribution locations. + #[serde(default)] + pub qemu_bridge_helper: String, /// TDX attestation/hash scheme policy. `legacy` keeps the existing /// digest.txt measurement path; `lite` opts into split measurement CBOR; @@ -384,6 +388,12 @@ pub struct CvmConfig { #[serde(default)] pub allowed_macvtap_parents: Vec, + /// Largest virtio-net queue pair count a deployment RPC caller may request. + /// There is no node-wide count for it to bind; lowering it below the + /// scaling cap does lower the vCPU-scaled default too. + #[serde(default = "default_max_net_queues")] + pub max_net_queues: u32, + /// Optional host-side filtering for bridge interfaces. This filter does /// not apply to macvtap interfaces. #[serde(default)] @@ -715,6 +725,19 @@ impl Config { } validate_networking(&self.cvm.networking)?; + // netd creates an unfiltered TAP when the filter name is empty, which + // is what unfiltered multiqueue bridges need. Libvirt mode must never + // reach that path: it would silently produce an unbound TAP where the + // operator asked for a filtered one. + anyhow::ensure!( + self.cvm.network_filter.mode != NetworkFilterMode::Libvirt + || !self.cvm.network_filter.filter.trim().is_empty(), + "cvm.network_filter.filter must not be empty when mode is libvirt" + ); + anyhow::ensure!( + (1..=MAX_NET_QUEUES).contains(&self.cvm.max_net_queues), + "cvm.max_net_queues must be between 1 and {MAX_NET_QUEUES}" + ); anyhow::ensure!( !self .cvm @@ -828,6 +851,21 @@ fn validate_networking(networking: &Networking) -> Result<()> { "cvm.networking.mac_prefix must contain 1 to 3 two-digit hexadecimal bytes" ); } + anyhow::ensure!( + networking.queues.is_none(), + "cvm.networking.queues has been removed; queue pairs default to the VM's vCPU count \ + (capped at {DEFAULT_MAX_NET_QUEUES}) and are overridden per deployment" + ); + // Both describe a per-VM entry's relationship to this configuration, so + // neither means anything on the node default itself. + anyhow::ensure!( + !networking.inherit_mode, + "cvm.networking.inherit_mode is per-deployment state and cannot be set on the node default" + ); + anyhow::ensure!( + networking.netd_interface.is_none(), + "cvm.networking.netd_interface is runtime state and cannot be set in configuration" + ); match networking.mode { NetworkingMode::Bridge => anyhow::ensure!( !networking.bridge.trim().is_empty(), @@ -850,6 +888,7 @@ fn validate_networking(networking: &Networking) -> Result<()> { "cvm.networking.macvtap_mode must be private, bridge, vepa, or passthru" ); } + // User mode has no identity fields of its own to check. NetworkingMode::User => {} } Ok(()) @@ -859,6 +898,26 @@ fn default_allowed_network_modes() -> Vec { vec![NetworkingMode::User, NetworkingMode::Bridge] } +fn default_max_net_queues() -> u32 { + DEFAULT_MAX_NET_QUEUES +} + +/// Where the vCPU-scaled default stops growing. Each queue pair costs a host +/// vhost thread and two MSI-X vectors, and cross-vCPU wakeups are expensive +/// under TDX, so the benefit runs out well before a large VM's vCPU count. +/// Raising `cvm.max_net_queues` lets a deployment ask for more; it does not +/// move this, because a bigger VM should not silently get a worse default. +pub const DEFAULT_QUEUE_SCALING_CAP: u32 = 16; + +/// Default ceiling on what a deployment RPC caller may request. +pub const DEFAULT_MAX_NET_QUEUES: u32 = 16; + +/// Hard bound on queue pairs from any source, well below anything QEMU or the +/// guest driver would refuse. It exists so a malformed or hostile request +/// cannot ask the host kernel for an unbounded device, not because 64 is a +/// property of virtio-net. +pub const MAX_NET_QUEUES: u32 = 64; + #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "lowercase")] pub enum NetworkingMode { @@ -871,7 +930,7 @@ pub enum NetworkingMode { /// Flat networking configuration. The `mode` field selects which backend is /// active; the remaining fields are only relevant for their respective mode /// and carry serde defaults so they can be omitted in the config file. -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] pub struct Networking { pub mode: NetworkingMode, @@ -908,6 +967,64 @@ pub struct Networking { // ── Custom fields ────────────────────────────────────────────── #[serde(default)] pub netdev: String, + + // ── Data plane tuning ────────────────────────────────────────── + /// Move packet processing from the QEMU main loop into the host kernel's + /// vhost-net data plane. `None` inherits the node default. Ignored by the + /// user-mode backend, which has no vhost support, and by custom mode, + /// which owns its whole netdev string. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vhost: Option, + /// virtio-net queue pairs. `None` scales with the VM's vCPU count. Only a + /// deployment sets this; there is no node-wide value, because the useful + /// number depends on the VM rather than the host. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queues: Option, + + // ── Ownership markers ────────────────────────────────────────── + /// Take `mode` from node configuration at every launch instead of from + /// this entry. + /// + /// A deployment that only tunes the data plane never named a backend, so + /// the node still owns which one this NIC uses. `mode` is not an `Option` + /// -- every consumer matches on it -- so the entry carries the node's + /// current mode and this flag says not to trust it across a node + /// configuration change. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub inherit_mode: bool, + /// What netd built for this NIC, recorded when it was built. + /// + /// Runtime state, like `device`: resolution always clears it. Teardown + /// reads this rather than re-deriving it from node configuration, because + /// an operator may change `network_filter.mode` or `max_net_queues` while + /// the VM runs, and what has to be removed is what was created. + #[serde(default, skip_serializing_if = "NetdInterface::is_none")] + pub netd_interface: NetdInterface, +} + +/// The host interface netd created for a NIC, if any. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum NetdInterface { + /// netd was not involved: user mode, custom mode, or a bridge NIC that + /// QEMU's own bridge helper attaches. + #[default] + None, + /// netd created the interface and bound no libvirt nwfilter to it. + Unfiltered, + /// netd created the interface and bound a libvirt nwfilter to it, which + /// removal has to delete before the interface goes away. + Filtered, +} + +impl NetdInterface { + pub fn is_none(&self) -> bool { + matches!(self, NetdInterface::None) + } + + pub fn is_filtered(&self) -> bool { + matches!(self, NetdInterface::Filtered) + } } impl Networking { @@ -915,6 +1032,66 @@ impl Networking { self.mode == NetworkingMode::Bridge } + /// Whether the vhost-net data plane applies to this interface. + /// + /// Defaults to enabled: the QEMU userspace backend drains every packet on + /// the single main-loop thread, which caps a CVM at one core's worth of + /// packet processing regardless of how many vCPUs it has. + pub fn vhost_enabled(&self) -> bool { + self.vhost.unwrap_or(true) && self.supports_vhost() + } + + /// Whether the backend selected by `mode` can carry a vhost-net data plane + /// at all. Custom mode is excluded because the operator supplies the whole + /// netdev string, including any vhost options. + pub fn supports_vhost(&self) -> bool { + Self::mode_supports_vhost(self.mode) + } + + /// The same question about a mode on its own, for a caller deciding + /// whether a request it has not built an entry for yet can be honoured. + pub fn mode_supports_vhost(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::Bridge | NetworkingMode::Macvtap) + } + + /// Whether the backend selected by `mode` can carry more than one queue + /// pair. + /// + /// Custom mode is excluded for the same reason as vhost: the operator + /// supplies the whole netdev string and the VMM cannot edit it, so a + /// multiqueue device line would have nothing to pair with. The RPC refuses + /// such a request, but a node that switches its default to custom must not + /// be able to produce one behind the RPC's back. + pub fn supports_multiqueue(&self) -> bool { + Self::mode_supports_multiqueue(self.mode) + } + + /// The same question about a mode on its own, for a caller deciding + /// whether a request it has not built an entry for yet can be honoured. + pub fn mode_supports_multiqueue(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::Bridge | NetworkingMode::Macvtap) + } + + /// Effective virtio-net queue pair count of a resolved NIC, never below + /// one. Resolution makes the vCPU-scaled default concrete, so an entry that + /// still carries none is read conservatively as single-queue. + pub fn queue_pairs(&self) -> u32 { + if !self.supports_multiqueue() { + return 1; + } + self.queues.unwrap_or(1).max(1) + } + + /// Queue pairs a VM with this many vCPUs gets when it asks for none. + /// + /// The guest driver uses at most one queue pair per vCPU, so the default + /// follows the vCPU count up to a fixed cap. A node that lowers + /// `max_net_queues` below that cap means it, so the default follows it + /// down; raising it above the cap only widens what a caller may request. + pub fn default_queue_pairs(vcpu: u32, max_net_queues: u32) -> u32 { + vcpu.clamp(1, DEFAULT_QUEUE_SCALING_CAP.min(max_net_queues).max(1)) + } + /// Parse the mac_prefix into bytes. Returns 0-3 bytes. pub fn mac_prefix_bytes(&self) -> Vec { if self.mac_prefix.is_empty() { @@ -1198,6 +1375,31 @@ mod tests { .expect("default VMM config should parse") } + /// The two ownership markers are additive on disk: manifests and runtime + /// network snapshots written before they existed still load, and an entry + /// that carries neither serializes exactly as it used to. + #[test] + fn ownership_markers_are_omitted_when_unset_and_default_when_absent() { + let mut networking: Networking = + serde_json::from_str(r#"{"mode":"bridge","bridge":"br0"}"#).unwrap(); + assert!(!networking.inherit_mode); + assert_eq!(networking.netd_interface, NetdInterface::None); + + let json = serde_json::to_string(&networking).unwrap(); + assert!(!json.contains("inherit_mode"), "{json}"); + assert!(!json.contains("netd_interface"), "{json}"); + + networking.inherit_mode = true; + networking.netd_interface = NetdInterface::Filtered; + let json = serde_json::to_string(&networking).unwrap(); + assert!(json.contains(r#""inherit_mode":true"#), "{json}"); + assert!(json.contains(r#""netd_interface":"filtered""#), "{json}"); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + networking + ); + } + #[test] fn config_validation_accepts_defaults() { let config = default_config(); @@ -1234,6 +1436,42 @@ mod tests { .to_string() .contains("range start")); + // An empty filter tells netd to create an unfiltered TAP, so libvirt + // mode must never carry one. + let mut config = default_config(); + config.cvm.network_filter.mode = NetworkFilterMode::Libvirt; + config.cvm.network_filter.filter = String::new(); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("network_filter.filter")); + + let mut config = default_config(); + config.cvm.max_net_queues = 0; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("max_net_queues")); + + let mut config = default_config(); + config.cvm.max_net_queues = MAX_NET_QUEUES + 1; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("max_net_queues")); + + // The node-wide value is gone; say so rather than ignoring it. + let mut config = default_config(); + config.cvm.networking.queues = Some(4); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("cvm.networking.queues has been removed")); + let mut config = default_config(); config.cvm.networking.mac_prefix = "02:not-hex".into(); assert!(config diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 931a373ae..a1e532c50 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -28,7 +28,7 @@ use crate::app::{ needs_swtpm, resolve_networking, validate_resolved_network, validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, }; -use crate::config::{CvmConfig, Networking, NetworkingMode}; +use crate::config::{CvmConfig, NetdInterface, Networking, NetworkingMode}; fn hex_sha256(data: &str) -> String { use sha2::Digest; @@ -354,16 +354,28 @@ fn networking_from_proto( cvm_config: &CvmConfig, ) -> Result> { let bridge = proto.bridge_name.trim().to_string(); - let mode = match proto.mode.as_str() { - "bridge" => NetworkingMode::Bridge, - "user" => NetworkingMode::User, - "macvtap" => NetworkingMode::Macvtap, - "" if bridge.is_empty() => return Ok(None), - "" => bail!("networking mode is required when bridge is set"), + let parent = proto.parent.trim().to_string(); + let tuned = proto.vhost.is_some() || proto.queues.is_some_and(|queues| queues != 0); + // Naming a bridge or a macvtap parent is naming a backend, and a backend + // needs a mode to go with it. Without one the entry would freeze whichever + // mode the node happened to have, then report a mode-less override still + // carrying a field only one mode accepts -- something nothing can send + // back once the node moves on. + let names_backend = !bridge.is_empty() || !parent.is_empty(); + // A request that only tunes the data plane keeps the node's backend. Node + // policy governs which backend a caller may *choose*, so inheriting one + // must not be denied by it. + let (mode, chosen) = match proto.mode.as_str() { + "bridge" => (NetworkingMode::Bridge, true), + "user" => (NetworkingMode::User, true), + "macvtap" => (NetworkingMode::Macvtap, true), + "" if !names_backend && !tuned => return Ok(None), + "" if !names_backend => (cvm_config.networking.mode, false), + "" => bail!("networking mode is required when a bridge or macvtap parent is set"), "custom" => bail!("custom networking mode is manifest-only"), other => bail!("unsupported networking mode '{other}'"), }; - if !cvm_config.allowed_network_modes.contains(&mode) { + if chosen && !cvm_config.allowed_network_modes.contains(&mode) { bail!( "networking mode '{}' is not allowed by node policy", proto.mode @@ -372,21 +384,66 @@ fn networking_from_proto( if mode != NetworkingMode::Bridge && !bridge.is_empty() { bail!("bridge_name is only valid for bridge networking mode"); } - if mode != NetworkingMode::Macvtap && !proto.parent.trim().is_empty() { + if mode != NetworkingMode::Macvtap && !parent.is_empty() { bail!("parent is only valid for macvtap networking mode"); } - if !proto.macvtap_mode.trim().is_empty() { + // `GetInfo` reports the resolved node defaults, and both vmm-cli and the + // web UI read that, change one field, and send the rest back. Naming a + // value the node would have supplied anyway therefore has to be accepted: + // leaving the field empty already yields exactly it, so echoing it grants + // nothing that policy was withholding. + // Node values are compared trimmed, the way the request's are: node + // configuration validation tolerates surrounding whitespace, and a value + // resolution would supply must not become unsendable over a space. + let node = &cvm_config.networking; + let macvtap_mode = proto.macvtap_mode.trim(); + if !macvtap_mode.is_empty() && macvtap_mode != node.macvtap_mode.trim() { bail!("macvtap_mode is node-controlled and cannot be set by deployment RPCs"); } - if !bridge.is_empty() && !cvm_config.allowed_bridges.contains(&bridge) { + if !bridge.is_empty() + && bridge != node.bridge.trim() + && !cvm_config.allowed_bridges.contains(&bridge) + { bail!("bridge_name '{bridge}' is not allowed by node policy"); } - let parent = proto.parent.trim().to_string(); - if !parent.is_empty() && !cvm_config.allowed_macvtap_parents.contains(&parent) { + if !parent.is_empty() + && parent != node.parent.trim() + && !cvm_config.allowed_macvtap_parents.contains(&parent) + { bail!("macvtap parent '{parent}' is not allowed by node policy"); } + // Same rule as the queue count below: a backend the caller chose and that + // has no vhost data plane is a request they can fix, so say so. An + // inherited one is not, and reads as off until the node moves. + if chosen && proto.vhost == Some(true) && !Networking::mode_supports_vhost(mode) { + bail!("{} networking has no vhost data plane", proto.mode); + } + // Queue pairs cost a host vhost thread and a pair of MSI-X vectors each, so + // the node caps what a deployment may ask for. + let queues = proto.queues.filter(|queues| *queues != 0); + if let Some(queues) = queues { + if queues > cvm_config.max_net_queues { + bail!( + "networking queues must not exceed {} on this node", + cvm_config.max_net_queues + ); + } + // Only a backend the caller *chose* is theirs to be wrong about. One + // they inherited can change under them -- that is the point of + // inheriting -- and refusing the request afterwards would strand the + // VM: GetInfo would keep reporting a queue count that nothing is + // allowed to send back. `queue_pairs()` already reads as one on both + // of these, so the request simply lies dormant until the node moves to + // a backend that can honour it. + if chosen && queues > 1 && !Networking::mode_supports_multiqueue(mode) { + bail!("{} networking does not support multiple queues", proto.mode); + } + } Ok(Some(Networking { mode, + // The caller named no backend, so the node keeps owning which one this + // NIC uses. `mode` above is only the node's current choice. + inherit_mode: !chosen, bridge, parent, // The forwarding mode is always inherited from node configuration. @@ -397,14 +454,84 @@ fn networking_from_proto( dhcp_start: String::new(), restrict: false, netdev: String::new(), + vhost: proto.vhost, + queues, + netd_interface: NetdInterface::None, })) } +/// Networking modes a client should offer. +/// +/// A mode the host could serve but node policy forbids is not one of them: +/// offering it puts a choice in the deploy dialog whose only outcome is "not +/// allowed by node policy", with nothing for the operator to do about it. A VM +/// can still be given a data plane override without naming any mode, which is +/// how a node whose own backend is not caller-selectable stays tunable. +fn advertised_modes(cvm_config: &CvmConfig, host_can_bridge: bool) -> Vec { + [ + (NetworkingMode::User, "user", true), + (NetworkingMode::Bridge, "bridge", host_can_bridge), + (NetworkingMode::Macvtap, "macvtap", true), + ] + .into_iter() + .filter(|(mode, _, host_supports)| { + *host_supports && cvm_config.allowed_network_modes.contains(mode) + }) + .map(|(_, name, _)| name.to_string()) + .collect() +} + +/// The node's policy, widened by what this VM's own NICs already pin. +/// +/// Deployment allowlists govern what a caller may *newly* select. A value one +/// of this VM's interfaces is already running on was selected when it was +/// still allowed, and it is reported back on every `GetInfo`; refusing it +/// would strand the VM rather than withhold anything. +fn held_networking_config(cvm_config: &CvmConfig, held: &[Networking]) -> CvmConfig { + let mut widened = cvm_config.clone(); + for networking in held { + if !networking.bridge.is_empty() { + widened.allowed_bridges.push(networking.bridge.clone()); + } + if !networking.parent.is_empty() { + widened + .allowed_macvtap_parents + .push(networking.parent.clone()); + } + } + widened +} + +/// A NIC that overrides nothing: whatever the node's `[cvm.networking]` says, +/// now and after the operator changes it. +fn node_default_networking(cvm_config: &CvmConfig) -> Networking { + Networking { + mode: cvm_config.networking.mode, + inherit_mode: true, + bridge: String::new(), + parent: String::new(), + macvtap_mode: String::new(), + device: String::new(), + mac_prefix: String::new(), + net: String::new(), + dhcp_start: String::new(), + restrict: false, + netdev: String::new(), + vhost: None, + queues: None, + netd_interface: NetdInterface::None, + } +} + fn network_from_required_proto( proto: &rpc::NetworkingConfig, cvm_config: &CvmConfig, ) -> Result { - networking_from_proto(proto, cvm_config)?.context("networking mode is required") + // An entry in a list that overrides nothing is not a missing mode: it is a + // NIC that follows the node entirely. Only the singular `networking` field + // can mean "no override at all", because there the absence is the message. + Ok(networking_from_proto(proto, cvm_config)? + .unwrap_or_else(|| node_default_networking(cvm_config))) } fn networks_from_proto( @@ -422,15 +549,48 @@ fn validate_default_network(cvm_config: &CvmConfig) -> Result<()> { } fn resolve_requested_networks( - networks: &[Networking], + requests: &[Networking], cvm_config: &CvmConfig, + vcpu: u32, ) -> Result> { - let resolved = networks + let merged = requests .iter() - .map(|networking| resolve_networking(networking, cvm_config)) + .map(|request| resolve_networking(request, cvm_config, vcpu)) .collect::>(); - validate_resolved_networks(&resolved)?; - Ok(resolved) + // Validate the merged view, because that is what the launch sees, then + // record the narrower view the manifest keeps. + validate_resolved_networks(&merged)?; + Ok(manifest_networks(merged, requests)) +} + +/// What a deployment records against the VM, given the merged view its launch +/// would see. +/// +/// Identity-bearing fields are pinned here so a VM keeps its address and bridge +/// for life. Everything else stays owned by the node, so a later change to +/// `[cvm.networking]` -- including an operator disabling vhost to roll the node +/// back -- still reaches VMs deployed with some other networking override. +fn manifest_networks(merged: Vec, requests: &[Networking]) -> Vec { + merged + .into_iter() + .zip(requests) + .map(|(mut entry, request)| { + if request.inherit_mode { + // The caller named no backend, so none of the node's matching + // identity fields are theirs to keep. + return request.clone(); + } + // Data plane tuning is not identity: leave what the caller did not + // ask for unset. + entry.vhost = request.vhost; + entry.queues = request.queues; + // Resolution always takes the forwarding mode from the node, so + // keeping the resolved value here would only report it back as + // though the VM owned it. + entry.macvtap_mode = String::new(); + entry + }) + .collect() } fn has_host_bridge_interface() -> bool { @@ -448,10 +608,10 @@ fn networks_from_vm_config( ) -> Result> { if !request.networks.is_empty() { let networks = networks_from_proto(&request.networks, cvm_config)?; - resolve_requested_networks(&networks, cvm_config) + resolve_requested_networks(&networks, cvm_config, request.vcpu) } else if let Some(networking) = request.networking.as_ref() { match networking_from_proto(networking, cvm_config)? { - Some(networking) => resolve_requested_networks(&[networking], cvm_config), + Some(networking) => resolve_requested_networks(&[networking], cvm_config, request.vcpu), None => Ok(vec![]), } } else { @@ -747,8 +907,14 @@ impl VmmRpc for RpcHandler { validate_default_network(&self.app.config.cvm)?; vec![] } else { - let networks = networks_from_proto(&request.networks, &self.app.config.cvm)?; - resolve_requested_networks(&networks, &self.app.config.cvm)? + // A bridge or parent this VM already holds is not a new grant. + // `GetInfo` keeps reporting it, and read-modify-write keeps + // sending it back, so refusing it once the node changes its own + // default would make the VM's configuration unsendable -- with + // no flag anywhere to clear a field the caller never typed. + let cvm = held_networking_config(&self.app.config.cvm, &manifest.networks); + let networks = networks_from_proto(&request.networks, &cvm)?; + resolve_requested_networks(&networks, &cvm, manifest.vcpu)? }; let is_running = self .app @@ -859,14 +1025,12 @@ impl VmmRpc for RpcHandler { } async fn get_meta(self) -> Result { - let mut supported_modes = vec!["user".to_string()]; let default_networking = &self.app.config.cvm.networking; let mut bridge_networking = default_networking.clone(); bridge_networking.mode = NetworkingMode::Bridge; - if validate_resolved_network(&bridge_networking).is_ok() || has_host_bridge_interface() { - supported_modes.push("bridge".to_string()); - } - supported_modes.push("macvtap".to_string()); + let host_can_bridge = + validate_resolved_network(&bridge_networking).is_ok() || has_host_bridge_interface(); + let supported_modes = advertised_modes(&self.app.config.cvm, host_can_bridge); Ok(GetMetaResponse { kms: Some(KmsSettings { url: self @@ -907,6 +1071,7 @@ impl VmmRpc for RpcHandler { NetworkingMode::Macvtap => "macvtap".to_string(), }, default_bridge: default_networking.bridge.clone(), + max_queues: self.app.config.cvm.max_net_queues, }), }) } @@ -1308,9 +1473,637 @@ mod tests { assert_eq!(networks[0].parent, "eth0"); assert!(networks[0].macvtap_mode.is_empty()); - let resolved = resolve_requested_networks(&networks, &cvm_config).unwrap(); - assert_eq!(resolved[0].parent, "eth0"); - assert_eq!(resolved[0].macvtap_mode, "private"); + // The manifest keeps the parent, which is identity, and not the + // forwarding mode, which the node owns and supplies at every launch. + let stored = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + assert_eq!(stored[0].parent, "eth0"); + assert!(stored[0].macvtap_mode.is_empty()); + + let at_launch = resolve_networking(&stored[0], &cvm_config, 4); + assert_eq!(at_launch.parent, "eth0"); + assert_eq!(at_launch.macvtap_mode, "private"); + + // Repointing the node's forwarding mode reaches the VM. + cvm_config.networking.macvtap_mode = "bridge".to_string(); + assert_eq!( + resolve_networking(&stored[0], &cvm_config, 4).macvtap_mode, + "bridge" + ); + } + + /// Node shapes a deployment can be reported against, each with the node + /// default fields that mode populates. + fn node_shapes() -> Vec<(&'static str, CvmConfig)> { + let mut bridge = test_cvm_config(); + bridge.networking.mode = NetworkingMode::Bridge; + bridge.networking.bridge = "br-node".into(); + + let mut macvtap = test_cvm_config(); + macvtap.networking.mode = NetworkingMode::Macvtap; + macvtap.networking.parent = "eth-node".into(); + macvtap.networking.macvtap_mode = "private".into(); + macvtap.allowed_network_modes.push(NetworkingMode::Macvtap); + + let user = test_cvm_config(); + vec![ + ("bridge node", bridge), + ("macvtap node", macvtap), + ("user node", user), + ] + } + + /// Every override a caller can express, including the tuning-only shape + /// that names no backend. + fn request_shapes(node: &CvmConfig) -> Vec<(String, rpc::NetworkingConfig)> { + let mode = networking_mode_name_for_test(node.networking.mode); + // The user-mode backend has neither multiqueue nor vhost, and naming it + // and then asking for either is a refusal rather than a round-trip + // failure. Asking to turn vhost off is always legal. + let multiqueue = Networking::mode_supports_multiqueue(node.networking.mode); + let vhost_on = Networking::mode_supports_vhost(node.networking.mode).then_some(true); + // Naming a backend's identity field is only legal alongside a mode, so + // it varies with the named case. The node's own values are used because + // that is what GetInfo reports back. + let identity: &[(&str, &str, &str)] = &[ + ("", "", ""), + ("+ bridge", node.networking.bridge.as_str(), ""), + ("+ parent", "", node.networking.parent.as_str()), + ]; + let mut shapes = vec![]; + for (named, mode) in [("named", mode.to_string()), ("inherited", String::new())] { + for (tuning, vhost, queues) in [ + ("untuned", None, None), + ("vhost off", Some(false), None), + ("queues", None, multiqueue.then_some(2)), + ("both", vhost_on, multiqueue.then_some(2)), + ] { + for (label, bridge, parent) in identity { + // An inherited entry may not name a backend at all, which + // is a refusal covered by its own test. + let inherited = mode.is_empty(); + if inherited && !(bridge.is_empty() && parent.is_empty()) { + continue; + } + // Only the mode that owns a field may carry it. + let bridge_ok = node.networking.mode == NetworkingMode::Bridge; + let parent_ok = node.networking.mode == NetworkingMode::Macvtap; + if (!bridge.is_empty() && !bridge_ok) || (!parent.is_empty() && !parent_ok) { + continue; + } + shapes.push(( + format!("{named} + {tuning} {label}"), + rpc::NetworkingConfig { + mode: mode.clone(), + bridge_name: bridge.to_string(), + parent: parent.to_string(), + vhost, + queues, + ..Default::default() + }, + )); + } + } + } + shapes + } + + /// A backend's identity field without a mode would freeze whichever mode + /// the node had at deploy time, and then be reported alongside an empty + /// mode -- a combination the RPC itself rejects, which would leave the VM's + /// tuning permanently uneditable. + #[test] + fn an_inherited_entry_may_not_name_a_backend() { + let mut cvm = test_cvm_config(); + cvm.networking.mode = NetworkingMode::Macvtap; + cvm.networking.parent = "eth-node".into(); + cvm.allowed_macvtap_parents.push("eth1".into()); + cvm.allowed_network_modes.push(NetworkingMode::Macvtap); + + let err = networking_from_proto( + &rpc::NetworkingConfig { + parent: "eth1".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap_err(); + assert!(err.to_string().contains("networking mode is required")); + + // Naming the mode alongside it is fine. + networking_from_proto( + &rpc::NetworkingConfig { + mode: "macvtap".into(), + parent: "eth1".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("a named backend is an override"); + } + + fn networking_mode_name_for_test(mode: NetworkingMode) -> &'static str { + match mode { + NetworkingMode::Bridge => "bridge", + NetworkingMode::User => "user", + NetworkingMode::Macvtap => "macvtap", + NetworkingMode::Custom => "custom", + } + } + + /// `GetInfo` reports the configuration that `UpdateVm` and `UpgradeApp` + /// take back, and both vmm-cli and the web UI read it, change one field, + /// and resend the rest. So everything reportable has to be acceptable, and + /// accepting it has to land on the same VM. + /// + /// This asserts the property over every mode and tuning combination on + /// purpose. Asserting one shape is how an inherited `parent` on a bridge + /// NIC, and then `macvtap_mode` and `bridge_name`, each reached a release: + /// every one of them was a case the fixed example did not cover. + #[test] + fn everything_get_info_reports_is_accepted_back_unchanged() { + for (node_label, cvm) in node_shapes() { + for (shape_label, request) in request_shapes(&cvm) { + let case = format!("{node_label} / {shape_label}"); + let Some(requested) = networking_from_proto(&request, &cvm) + .unwrap_or_else(|error| panic!("{case}: deployment rejected: {error:#}")) + else { + // No override at all; nothing is recorded, nothing to report. + continue; + }; + // Skip the host-dependent bridge existence check: this is + // about what the RPC reports versus what it accepts. + let merged = resolve_networking(&requested, &cvm, 4); + let stored = manifest_networks(vec![merged.clone()], &[requested]); + + let reported = crate::app::networking_to_proto(&stored[0]); + let accepted = networking_from_proto(&reported, &cvm) + .unwrap_or_else(|error| { + panic!("{case}: GetInfo output was rejected on the way back: {error:#}") + }) + .unwrap_or_else(|| panic!("{case}: the override was lost in the round trip")); + + let restored = + manifest_networks(vec![resolve_networking(&accepted, &cvm, 4)], &[accepted]); + assert_eq!(stored, restored, "{case}: round trip changed the manifest"); + assert_eq!( + resolve_networking(&restored[0], &cvm, 4), + merged, + "{case}: round trip changed what the launch sees" + ); + } + } + } + + /// A VM must not become uneditable because its node moved somewhere its + /// tuning does not apply. The report and the deployment RPC have to agree + /// on every backend the node can be pointed at, not just the one it had + /// when the VM was deployed. + #[test] + fn an_inherited_override_still_round_trips_after_the_node_moves() { + let mut cvm = test_cvm_config(); + cvm.networking.mode = NetworkingMode::Bridge; + cvm.networking.bridge = "br-node".into(); + + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(4), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("tuning must produce an override"); + let stored = manifest_networks(vec![resolve_networking(&requested, &cvm, 8)], &[requested]); + + for mode in [ + NetworkingMode::User, + NetworkingMode::Custom, + NetworkingMode::Macvtap, + NetworkingMode::Bridge, + ] { + cvm.networking.mode = mode; + cvm.networking.parent = "eth-node".into(); + cvm.networking.netdev = "tap,id=net0,ifname=custom0".into(); + + let reported = crate::app::networking_to_proto(&stored[0]); + let accepted = networking_from_proto(&reported, &cvm) + .unwrap_or_else(|error| panic!("{mode:?}: report was rejected: {error:#}")) + .unwrap_or_else(|| panic!("{mode:?}: the override was lost")); + let restored = + manifest_networks(vec![resolve_networking(&accepted, &cvm, 8)], &[accepted]); + // An inherited entry still has to hold *some* mode -- every + // consumer matches on one -- and it is rewritten to whatever the + // node has now. Resolution ignores it, so compare what the launch + // sees rather than the field nothing reads. + assert!(restored[0].inherit_mode, "{mode:?}: pinned a backend"); + assert_eq!( + resolve_networking(&restored[0], &cvm, 8), + resolve_networking(&stored[0], &cvm, 8), + "{mode:?}: round trip changed what the launch sees" + ); + assert_eq!(restored[0].queues, Some(4), "{mode:?}: lost the request"); + } + } + + /// The counterpart: a node that changes its mind still reaches VMs that + /// never named a backend, and never reaches ones that did. + #[test] + fn a_node_backend_change_reaches_exactly_the_vms_that_inherited_it() { + let mut cvm = test_cvm_config(); + cvm.networking.mode = NetworkingMode::Bridge; + cvm.networking.bridge = "br-node".into(); + + let tuning_only = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("tuning must produce an override"); + let named = networking_from_proto( + &rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("a named backend is an override"); + + let requests = vec![tuning_only, named]; + let merged = requests + .iter() + .map(|request| resolve_networking(request, &cvm, 4)) + .collect::>(); + let stored = manifest_networks(merged, &requests); + + // The operator repoints the node at a different backend. + cvm.networking.mode = NetworkingMode::User; + assert_eq!( + resolve_networking(&stored[0], &cvm, 4).mode, + NetworkingMode::User, + "a VM that never named a backend must follow the node" + ); + assert_eq!( + resolve_networking(&stored[1], &cvm, 4).mode, + NetworkingMode::Bridge, + "a VM that named its backend keeps it for life" + ); + // User mode has no multiqueue backend, so the inherited NIC drops to + // one queue pair while it is there -- but the request is not lost. + assert_eq!(resolve_networking(&stored[0], &cvm, 4).queue_pairs(), 1); + assert_eq!(stored[0].queues, Some(2)); + cvm.networking.mode = NetworkingMode::Bridge; + assert_eq!( + resolve_networking(&stored[0], &cvm, 4).queue_pairs(), + 2, + "tuning must survive an excursion through a backend that ignores it" + ); + } + + #[test] + fn queue_requests_are_bounded_by_node_policy() { + let mut cvm_config = test_cvm_config(); + cvm_config.max_net_queues = 4; + cvm_config.allowed_bridges.push("tenant-br0".to_string()); + let request = |queues: u32| { + [rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "tenant-br0".to_string(), + queues: Some(queues), + ..Default::default() + }] + }; + + let networks = networks_from_proto(&request(4), &cvm_config).unwrap(); + assert_eq!(networks[0].queues, Some(4)); + assert_eq!(networks[0].queue_pairs(), 4); + + let err = networks_from_proto(&request(5), &cvm_config).unwrap_err(); + assert!(err.to_string().contains("must not exceed 4")); + } + + /// A mode the deploy dialog offers has to be one the deployment RPC will + /// take. Offering one node policy forbids puts a choice in front of an + /// operator whose only outcome is "not allowed by node policy". + #[test] + fn advertised_modes_are_ones_the_rpc_would_accept() { + let mut cvm = test_cvm_config(); + cvm.allowed_network_modes = vec![NetworkingMode::User]; + for mode in ["bridge", "macvtap"] { + assert!( + networking_from_proto( + &rpc::NetworkingConfig { + mode: mode.to_string(), + ..Default::default() + }, + &cvm, + ) + .is_err(), + "{mode} should be refused by this policy" + ); + } + assert_eq!(advertised_modes(&cvm, true), vec!["user".to_string()]); + + cvm.allowed_network_modes = vec![NetworkingMode::User, NetworkingMode::Macvtap]; + assert_eq!( + advertised_modes(&cvm, true), + vec!["user".to_string(), "macvtap".to_string()] + ); + + // A mode policy allows but the host cannot serve is still not offered. + cvm.allowed_network_modes.push(NetworkingMode::Bridge); + assert!(!advertised_modes(&cvm, false).contains(&"bridge".to_string())); + assert!(advertised_modes(&cvm, true).contains(&"bridge".to_string())); + } + + /// A VM keeps its bridge for life, so the node dropping that bridge from + /// its own configuration must not make the VM's reported configuration + /// unsendable -- there is no flag anywhere to clear a field the caller + /// never typed. + #[test] + fn a_vm_may_restate_a_bridge_it_already_holds() { + let mut cvm = test_cvm_config(); + cvm.networking.mode = NetworkingMode::Bridge; + cvm.networking.bridge = "br-new".into(); + assert!(cvm.allowed_bridges.is_empty()); + + let held = [Networking { + bridge: "br-old".into(), + ..node_default_networking(&cvm) + }]; + let request = [rpc::NetworkingConfig { + mode: "bridge".into(), + bridge_name: "br-old".into(), + queues: Some(2), + ..Default::default() + }]; + + // Without the VM's own holdings this is a bridge it may not select. + let err = networks_from_proto(&request, &cvm).unwrap_err(); + assert!(err.to_string().contains("not allowed by node policy")); + + let widened = held_networking_config(&cvm, &held); + let networks = networks_from_proto(&request, &widened).unwrap(); + assert_eq!(networks[0].bridge, "br-old"); + + // And it is still only this VM's own values that are permitted. + let other = [rpc::NetworkingConfig { + mode: "bridge".into(), + bridge_name: "br-someone-else".into(), + ..Default::default() + }]; + assert!(networks_from_proto(&other, &widened).is_err()); + } + + /// vhost follows the same rule as the queue count: refused for a backend + /// the caller chose and that has none, accepted and dormant for one they + /// inherited. Accepting it silently on a chosen backend would leave the + /// deploy dialog reporting `vhost: on` next to a NIC running without it. + #[test] + fn vhost_is_refused_only_for_a_backend_the_caller_chose() { + let cvm_config = test_cvm_config(); + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + vhost: Some(true), + ..Default::default() + }, + &cvm_config, + ) + .unwrap_err(); + assert!(err.to_string().contains("no vhost data plane"), "{err:#}"); + + // Turning it off is a no-op that matches reality, so it is allowed. + networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + vhost: Some(false), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + + // Inherited from a user-mode node: accepted, dormant, and live again + // when the node moves to a backend that has one. + let mut cvm_config = test_cvm_config(); + assert_eq!(cvm_config.networking.mode, NetworkingMode::User); + let requested = networking_from_proto( + &rpc::NetworkingConfig { + vhost: Some(true), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert!(!resolve_networking(&requested, &cvm_config, 4).vhost_enabled()); + + cvm_config.networking.mode = NetworkingMode::Bridge; + cvm_config.networking.bridge = "br-node".into(); + assert!(resolve_networking(&requested, &cvm_config, 4).vhost_enabled()); + } + + /// A queue count is refused for a backend the caller chose and that cannot + /// honour it, because the caller can fix the request. It is accepted for + /// one they inherited, because they cannot: the node picked that backend + /// and may pick another tomorrow, and refusing would leave GetInfo + /// reporting a count nothing is allowed to send back. + #[test] + fn a_queue_count_is_refused_only_for_a_backend_the_caller_chose() { + let cvm_config = test_cvm_config(); + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + queues: Some(4), + ..Default::default() + }, + &cvm_config, + ) + .unwrap_err(); + assert!(err.to_string().contains("does not support multiple queues")); + + // Inherited: accepted, and dormant until the node moves to a backend + // that can honour it. + for mode in [NetworkingMode::Custom, NetworkingMode::User] { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.mode = mode; + cvm_config.networking.netdev = "tap,id=net0,ifname=custom0".into(); + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(4), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert_eq!(requested.queues, Some(4)); + assert_eq!( + resolve_networking(&requested, &cvm_config, 8).queue_pairs(), + 1, + "{mode:?} cannot carry multiqueue, whatever was asked for" + ); + + // And the request is still there when the node moves back. + cvm_config.networking.mode = NetworkingMode::Bridge; + cvm_config.networking.bridge = "br-node".into(); + assert_eq!( + resolve_networking(&requested, &cvm_config, 8).queue_pairs(), + 4 + ); + } + } + + #[test] + fn user_mode_rejects_multiqueue_but_a_single_queue_is_fine() { + let cvm_config = test_cvm_config(); + let request = |queues: u32| { + [rpc::NetworkingConfig { + mode: "user".to_string(), + queues: Some(queues), + ..Default::default() + }] + }; + + networks_from_proto(&request(1), &cvm_config).unwrap(); + let err = networks_from_proto(&request(2), &cvm_config).unwrap_err(); + assert!(err.to_string().contains("does not support multiple queues")); + } + + #[test] + fn tuning_alone_keeps_the_node_backend_without_tripping_mode_policy() { + let mut cvm_config = test_cvm_config(); + // A backend the node uses but does not let callers choose. + cvm_config.networking.mode = NetworkingMode::Macvtap; + cvm_config.networking.parent = "eth0".to_string(); + assert!(!cvm_config + .allowed_network_modes + .contains(&NetworkingMode::Macvtap)); + + let networking = networking_from_proto( + &rpc::NetworkingConfig { + vhost: Some(false), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert_eq!(networking.mode, NetworkingMode::Macvtap); + assert_eq!(networking.vhost, Some(false)); + + // Naming that backend explicitly is still a choice, and still denied. + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "macvtap".to_string(), + vhost: Some(false), + ..Default::default() + }, + &cvm_config, + ) + .unwrap_err(); + assert!(err.to_string().contains("not allowed by node policy")); + + // An untouched request still means "no override at all". + assert!( + networking_from_proto(&rpc::NetworkingConfig::default(), &cvm_config) + .unwrap() + .is_none() + ); + } + + #[test] + fn an_inherited_backend_is_never_pinned_into_the_manifest() { + // Tuning must not become a way to pin a backend the caller was never + // allowed to choose, nor to freeze one the node still owns. + let mut cvm_config = test_cvm_config(); + cvm_config.networking.mode = NetworkingMode::Macvtap; + cvm_config.networking.parent = "eth0".to_string(); + cvm_config.networking.macvtap_mode = "private".to_string(); + + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(2), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert!(requested.inherit_mode); + + let persisted = resolve_requested_networks(&[requested], &cvm_config, 4).unwrap(); + assert_eq!(persisted[0].queues, Some(2)); + // Nothing the node owns was copied in. + assert!(persisted[0].parent.is_empty()); + assert!(persisted[0].macvtap_mode.is_empty()); + assert!(persisted[0].net.is_empty()); + + // Repointing the node moves the VM with it. + cvm_config.networking.parent = "eth1".to_string(); + let at_launch = resolve_networking(&persisted[0], &cvm_config, 4); + assert_eq!(at_launch.parent, "eth1"); + assert_eq!(at_launch.queue_pairs(), 2); + } + + #[test] + fn deployment_pins_identity_but_not_data_plane_tuning() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.vhost = Some(true); + cvm_config.networking.queues = Some(2); + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "user".to_string(), + vhost: Some(false), + ..Default::default() + }], + &cvm_config, + ) + .unwrap(); + + let resolved = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + // Identity-bearing fields are resolved and kept. + assert!(!resolved[0].net.is_empty()); + // The explicit request is kept. + assert_eq!(resolved[0].vhost, Some(false)); + // What the caller never asked for stays unset, so the node still owns + // it: an operator disabling vhost node-wide must reach this VM too. + assert_eq!(resolved[0].queues, None); + } + + #[test] + fn a_node_wide_vhost_rollback_reaches_a_vm_deployed_with_an_override() { + // macvtap keeps this independent of which interfaces the test host has. + let mut cvm_config = test_cvm_config(); + cvm_config + .allowed_network_modes + .push(NetworkingMode::Macvtap); + cvm_config.allowed_macvtap_parents.push("eth0".to_string()); + cvm_config.networking.parent = "eth0".to_string(); + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "macvtap".to_string(), + parent: "eth0".to_string(), + ..Default::default() + }], + &cvm_config, + ) + .unwrap(); + let persisted = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + assert!(persisted[0].vhost.is_none()); + + cvm_config.networking.vhost = Some(false); + let at_launch = resolve_networking(&persisted[0], &cvm_config, 4); + assert!(!at_launch.vhost_enabled()); } #[test] @@ -1399,18 +2192,35 @@ mod tests { } #[test] - fn repeated_networks_rejects_empty_entries() { - let err = networks_from_proto( + /// An entry in a list that overrides nothing describes a NIC that follows + /// the node entirely -- which is a thing an operator can mean, and the + /// only way the web UI can leave a NIC's backend unpinned. Rejecting it + /// would make an inherited NIC uneditable the moment its tuning is cleared. + fn repeated_networks_accepts_an_entry_that_overrides_nothing() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.mode = NetworkingMode::Bridge; + cvm_config.networking.bridge = "br-node".into(); + + let networks = networks_from_proto( &[rpc::NetworkingConfig { mode: String::new(), bridge_name: String::new(), ..Default::default() }], - &test_cvm_config(), + &cvm_config, ) - .unwrap_err(); + .unwrap(); + assert_eq!(networks.len(), 1); + assert!(networks[0].inherit_mode); + assert_eq!(networks[0].queues, None); + assert!(networks[0].bridge.is_empty()); - assert!(err.to_string().contains("networking mode is required")); + // It follows the node, like a VM with no networks at all. + cvm_config.networking.mode = NetworkingMode::User; + assert_eq!( + resolve_networking(&networks[0], &cvm_config, 4).mode, + NetworkingMode::User + ); } #[test] diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index b7a946133..f58b73358 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -29,7 +29,7 @@ use tokio::{ net::{UnixListener, UnixStream}, time::timeout, }; -use tracing::{info, warn}; +use tracing::{debug, info, warn}; use uuid::Uuid; use wait_timeout::ChildExt; @@ -41,6 +41,9 @@ const COMMAND_TIMEOUT: Duration = Duration::from_secs(30); const IP_PATH: &str = "/usr/sbin/ip"; const VIRSH_PATH: &str = "/usr/bin/virsh"; const LOCK_PATH: &str = "/run/lock/dstack-netd.lock"; +/// Upper bound on TAP queue pairs netd will create. Mirrors the VMM's own cap +/// so a malformed request cannot ask the kernel for an unbounded device. +const MAX_QUEUES: u32 = 64; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InterfaceIdentity { @@ -56,9 +59,17 @@ pub struct PrepareBridgeRequest { pub bridge: String, pub mac: String, pub qemu_uid: u32, + /// libvirt nwfilter to bind. Empty creates an unfiltered TAP, which is what + /// multiqueue bridge networking needs on nodes without libvirt. + #[serde(default)] pub filter: String, #[serde(default)] pub parameters: BTreeMap, + /// virtio-net queue pairs. Zero or one creates a single-queue TAP. QEMU + /// rejects a device whose `IFF_MULTI_QUEUE` state differs from its own + /// `queues=` argument, so this must match the launch exactly. + #[serde(default)] + pub queues: u32, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -70,6 +81,10 @@ pub struct PrepareMacvtapRequest { pub qemu_uid: u32, #[serde(default)] pub mode: String, + /// virtio-net queue pairs. The device is created with matching hardware + /// queues; QEMU then opens the character device once per queue. + #[serde(default)] + pub queues: u32, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -80,15 +95,27 @@ pub enum Request { Remove { #[serde(flatten)] identity: InterfaceIdentity, + /// Whether this interface was created with an nwfilter binding. + /// Defaults to true because every netd TAP carried one before + /// unfiltered multiqueue TAPs existed, so an older VMM still gets its + /// bindings cleaned up. + #[serde(default = "default_filtered")] + filtered: bool, }, /// Verify a deterministic TAP and binding for operations and integration /// diagnostics. The VMM startup path uses Prepare rather than Check. Check { #[serde(flatten)] identity: InterfaceIdentity, + #[serde(default = "default_filtered")] + filtered: bool, }, } +fn default_filtered() -> bool { + true +} + #[derive(Debug, Serialize, Deserialize)] struct Response { ok: bool, @@ -96,10 +123,33 @@ struct Response { tap: Option, #[serde(default, skip_serializing_if = "Option::is_none")] device: Option, + /// Queue pairs the interface was actually created with. Absent from a netd + /// that predates multiqueue, which is how the VMM tells the difference + /// between "one queue was requested" and "this netd ignored the request". + #[serde(default, skip_serializing_if = "Option::is_none")] + queues: Option, #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, } +/// What netd built, echoed back so the caller can verify it matches the +/// request before handing the interface to QEMU. +struct Prepared { + tap: String, + device: Option, + queues: Option, +} + +impl Prepared { + fn tap(tap: String) -> Self { + Self { + tap, + device: None, + queues: None, + } + } +} + pub fn tap_name(identity: &InterfaceIdentity) -> String { let input = format!( "{}\0{}\0{}", @@ -119,6 +169,7 @@ pub fn instance_id(configured: &str, run_path: &Path) -> String { pub struct PreparedInterface { pub device: Option, + pub queues: Option, } pub async fn request(socket: &Path, request: &Request) -> Result { @@ -154,11 +205,10 @@ pub async fn request(socket: &Path, request: &Request) -> Result Result { async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Result<()> { // Access is authorized by the Unix socket's owner, group, and mode. Any // process that can connect is trusted with the complete netd protocol. - let response = match read_request(stream) - .await - .and_then(|request| handle_request(&config.libvirt_uri, request)) - { - Ok((tap, device)) => Response { + let outcome = match read_request(stream).await { + // The VMM checks whether netd is reachable by connecting, because a + // netd that died leaves its socket behind. Answering a probe with a + // parse error and a warning would fill the log with reports of the + // VMM working correctly. + Ok(None) => { + debug!("netd liveness probe"); + return Ok(()); + } + Ok(Some(request)) => handle_request(&config.libvirt_uri, request), + // A request that arrived but could not be understood still gets an + // answer. A VMM newer than this netd sends operations it does not + // know, and "unknown variant `prepare_foo`" is what tells the operator + // to upgrade; a closed connection tells them nothing. + Err(error) => Err(error), + }; + let response = match outcome { + Ok(prepared) => Response { ok: true, - tap: Some(tap), - device, + tap: Some(prepared.tap), + device: prepared.device, + queues: prepared.queues, error: None, }, Err(error) => { @@ -241,6 +305,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul ok: false, tap: None, device: None, + queues: None, error: Some(format!("{error:#}")), } } @@ -251,77 +316,82 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul Ok(()) } -async fn read_request(stream: &mut UnixStream) -> Result { +/// Reads one request, or `None` if the peer closed without sending anything. +async fn read_request(stream: &mut UnixStream) -> Result> { let mut message = Vec::new(); stream .take(MAX_MESSAGE_SIZE + 1) .read_to_end(&mut message) .await?; + if message.is_empty() { + return Ok(None); + } if message.len() as u64 > MAX_MESSAGE_SIZE { bail!("request exceeds {MAX_MESSAGE_SIZE} bytes"); } - serde_json::from_slice(&message).context("invalid netd request") + serde_json::from_slice(&message) + .map(Some) + .context("invalid netd request") } -fn handle_request(libvirt_uri: &str, request: Request) -> Result<(String, Option)> { +fn handle_request(libvirt_uri: &str, request: Request) -> Result { let _lock = OperationLock::acquire()?; match request { - Request::PrepareBridge(request) => { - prepare_bridge(libvirt_uri, &request).map(|tap| (tap, None)) - } - Request::PrepareMacvtap(request) => prepare_macvtap( - libvirt_uri, - &request.identity, - &request.parent, - &request.mac, - request.qemu_uid, - &request.mode, - ) - .map(|(tap, device)| (tap, Some(device))), - Request::Remove { identity } => { + Request::PrepareBridge(request) => prepare_bridge(libvirt_uri, &request), + Request::PrepareMacvtap(request) => prepare_macvtap(libvirt_uri, &request), + Request::Remove { identity, filtered } => { validate_identity(&identity)?; let tap = tap_name(&identity); - remove_interface(libvirt_uri, &tap)?; - Ok((tap, None)) + remove_interface(libvirt_uri, &tap, binding_cleanup(filtered))?; + Ok(Prepared::tap(tap)) } - Request::Check { identity } => { + Request::Check { identity, filtered } => { validate_identity(&identity)?; let tap = tap_name(&identity); if !Path::new("/sys/class/net").join(&tap).exists() { bail!("TAP {tap} does not exist"); } - if !is_macvtap(&tap) { + // An unfiltered TAP has no binding to dump; asking for one would + // report a healthy multiqueue interface as broken. + if filtered && !is_macvtap(&tap) { virsh(libvirt_uri, &["nwfilter-binding-dumpxml", &tap], None)?; } - Ok((tap, None)) + Ok(Prepared::tap(tap)) } } } -fn prepare_macvtap( - libvirt_uri: &str, - identity: &InterfaceIdentity, - parent: &str, - mac: &str, - qemu_uid: u32, - mode: &str, -) -> Result<(String, String)> { +fn prepare_macvtap(libvirt_uri: &str, request: &PrepareMacvtapRequest) -> Result { + let identity = &request.identity; + let parent = request.parent.as_str(); + let qemu_uid = request.qemu_uid; validate_identity(identity)?; validate_name("parent", parent, 15, "_.-")?; if !Path::new("/sys/class/net").join(parent).exists() { bail!("parent interface {parent} does not exist"); } - validate_mac(mac)?; - let mode = if mode.is_empty() { "private" } else { mode }; + validate_mac(&request.mac)?; + let mac = request.mac.as_str(); + let mode = if request.mode.is_empty() { + "private" + } else { + request.mode.as_str() + }; if !matches!(mode, "private" | "bridge" | "vepa" | "passthru") { bail!("invalid macvtap mode"); } + let queues = validate_queues(request.queues)?; let tap = tap_name(identity); - remove_interface(libvirt_uri, &tap)?; - ip(&[ - "link", "add", "link", parent, "name", &tap, "address", mac, "type", "macvtap", "mode", - mode, - ])?; + remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; + let queue_count = queues.to_string(); + let mut add = vec!["link", "add", "link", parent, "name", &tap, "address", mac]; + if queues > 1 { + // macvtap defaults to a single hardware queue pair. Without this the + // extra tap queues exist but the lower device still serializes. + add.extend_from_slice(&["numtxqueues", &queue_count, "numrxqueues", &queue_count]); + } + add.extend_from_slice(&["type", "macvtap", "mode", mode]); + ip(&add)?; let result = (|| { let ifindex = std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifindex")) @@ -346,11 +416,15 @@ fn prepare_macvtap( })(); match result { Ok(device) => { - info!(%tap, %parent, %mode, %device, "prepared macvtap"); - Ok((tap, device)) + info!(%tap, %parent, %mode, %device, %queues, "prepared macvtap"); + Ok(Prepared { + tap, + device: Some(device), + queues: Some(queues), + }) } Err(error) => { - let _ = remove_interface(libvirt_uri, &tap); + let _ = remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort); Err(error) } } @@ -384,41 +458,83 @@ impl Drop for OperationLock { } } -fn prepare_bridge(libvirt_uri: &str, request: &PrepareBridgeRequest) -> Result { +fn prepare_bridge(libvirt_uri: &str, request: &PrepareBridgeRequest) -> Result { validate_prepare_bridge(request)?; + let filtered = !request.filter.is_empty(); let tap = tap_name(&request.identity); // A failed VMM start may leave a deterministic resource behind. Replacing // it makes prepare idempotent without accepting a caller-selected TAP. - remove_interface(libvirt_uri, &tap)?; + remove_interface(libvirt_uri, &tap, binding_cleanup(filtered))?; let uid = request.qemu_uid.to_string(); - ip(&["tuntap", "add", "dev", &tap, "mode", "tap", "user", &uid])?; + let queues = validate_queues(request.queues)?; + let mut add = vec!["tuntap", "add", "dev", &tap, "mode", "tap"]; + if queues > 1 { + // QEMU refuses to attach when the device's IFF_MULTI_QUEUE state does + // not match its own `queues=` argument, in either direction. + add.push("multi_queue"); + } + add.extend_from_slice(&["user", &uid]); + ip(&add)?; let result = (|| { ip(&["link", "set", "dev", &tap, "master", &request.bridge])?; - let xml = binding_xml(request, &tap); - virsh( - libvirt_uri, - &["nwfilter-binding-create", "--validate", "/dev/stdin"], - Some(xml.as_bytes()), - )?; + if filtered { + let xml = binding_xml(request, &tap); + virsh( + libvirt_uri, + &["nwfilter-binding-create", "--validate", "/dev/stdin"], + Some(xml.as_bytes()), + )?; + } ip(&["link", "set", "dev", &tap, "up"])?; Ok(()) })(); if let Err(error) = result { - let _ = remove_interface(libvirt_uri, &tap); + let _ = remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort); return Err(error); } - info!(%tap, bridge = %request.bridge, filter = %request.filter, "prepared filtered TAP"); - Ok(tap) + info!(%tap, bridge = %request.bridge, filter = %request.filter, %queues, "prepared TAP"); + Ok(Prepared { + tap, + device: None, + queues: Some(queues), + }) } -fn remove_interface(libvirt_uri: &str, tap: &str) -> Result<()> { +/// How hard removal must try to clear an nwfilter binding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BindingCleanup { + /// The binding must be gone before this returns, because the caller is + /// about to create one at the same interface name and libvirt refuses a + /// duplicate. + Required, + /// Delete a binding if libvirt can be reached, but do not fail the removal + /// when it cannot. Unfiltered TAPs live on nodes where `libvirtd` need not + /// be running at all, and a stale binding left by an earlier, filtered + /// interface at this name is still worth clearing when it is. + BestEffort, +} + +fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Result<()> { let macvtap = is_macvtap(tap); if Path::new("/sys/class/net").join(tap).exists() { let _ = ip(&["link", "set", "dev", tap, "down"]); } + // A macvtap interface never carries a binding. Anything else might: this + // name may have been a filtered bridge TAP before, and the binding + // outlives the interface. if !macvtap { - delete_binding(libvirt_uri, tap)?; + match cleanup { + BindingCleanup::Required => delete_binding(libvirt_uri, tap)?, + BindingCleanup::BestEffort => { + // netd refuses to start without virsh, so the binary is always + // here; libvirtd need not be running, and on a node that only + // wants macvtap or multiqueue it usually is not. + if let Err(error) = delete_binding(libvirt_uri, tap) { + warn!(%tap, "could not clear a possible nwfilter binding: {error:#}"); + } + } + } } if Path::new("/sys/class/net").join(tap).exists() { ip(&["link", "delete", "dev", tap])?; @@ -434,22 +550,26 @@ fn is_macvtap(interface: &str) -> bool { .exists() } +/// Deletes an interface's nwfilter binding, if it has one. +/// +/// Goes through the same `COMMAND_TIMEOUT`-bounded helper as every other virsh +/// call. netd's accept loop is strictly serialized, so an unbounded call here +/// would let one unreachable libvirt stall every other VM's prepare and remove. fn delete_binding(uri: &str, tap: &str) -> Result<()> { - let output = Command::new(VIRSH_PATH) - .args(["--connect", uri, "nwfilter-binding-delete", tap]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .context("failed to execute virsh")?; - if output.status.success() { - return Ok(()); - } - let error = String::from_utf8_lossy(&output.stderr); - if error.contains("Network filter binding not found") { - return Ok(()); + match virsh(uri, &["nwfilter-binding-delete", tap], None) { + Ok(()) => Ok(()), + // Removal is idempotent. Having no binding is the normal case for + // macvtap, for unfiltered multiqueue TAPs, and for any name being + // reused after an earlier removal already cleared it. + Err(error) + if error + .to_string() + .contains("Network filter binding not found") => + { + Ok(()) + } + Err(error) => Err(error).context(format!("virsh failed to delete binding {tap}")), } - bail!("virsh failed to delete binding {tap}: {}", error.trim()) } fn binding_xml(request: &PrepareBridgeRequest, tap: &str) -> String { @@ -505,7 +625,9 @@ fn validate_prepare_bridge(request: &PrepareBridgeRequest) -> Result<()> { bail!("{} is not a host bridge", request.bridge); } validate_mac(&request.mac)?; - validate_name("filter", &request.filter, 128, "_.:-")?; + if !request.filter.is_empty() { + validate_name("filter", &request.filter, 128, "_.:-")?; + } if request.parameters.len() > 64 { bail!("too many nwfilter parameters"); } @@ -533,6 +655,25 @@ fn validate_identity(identity: &InterfaceIdentity) -> Result<()> { Ok(()) } +/// A caller that knows a binding is there needs it gone; one that does not +/// still clears whatever it finds, without failing when libvirt is absent. +fn binding_cleanup(filtered: bool) -> BindingCleanup { + if filtered { + BindingCleanup::Required + } else { + BindingCleanup::BestEffort + } +} + +/// Normalizes a requested queue pair count. Zero means the caller did not ask +/// for multiqueue, which is the same device shape as one queue pair. +fn validate_queues(queues: u32) -> Result { + if queues > MAX_QUEUES { + bail!("queues must not exceed {MAX_QUEUES}"); + } + Ok(queues.max(1)) +} + fn validate_name(label: &str, value: &str, max: usize, punctuation: &str) -> Result<()> { if value.is_empty() || value.len() > max @@ -675,6 +816,7 @@ mod tests { qemu_uid: 1000, filter: "clean-traffic".into(), parameters: BTreeMap::from([("IP".into(), "10.0.0.2<&".into())]), + queues: 0, }; let xml = binding_xml(&request, "dt123"); assert!(xml.contains("instance<&")); @@ -693,6 +835,7 @@ mod tests { fn remove_protocol_keeps_identity_fields_flat() { let request = Request::Remove { identity: identity("instance", "vm", 2), + filtered: true, }; let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "remove"); @@ -711,6 +854,7 @@ mod tests { qemu_uid: 1000, filter: "clean-traffic".into(), parameters: BTreeMap::new(), + queues: 0, }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -719,6 +863,86 @@ mod tests { assert!(value.get("identity").is_none()); } + #[test] + fn removal_defaults_to_filtered_so_older_vmms_still_drop_bindings() { + let decoded: Request = serde_json::from_value(serde_json::json!({ + "operation": "remove", + "instance_id": "instance", + "vm_id": "vm", + "nic_index": 0, + })) + .unwrap(); + let Request::Remove { filtered, .. } = decoded else { + panic!("wrong variant"); + }; + assert!(filtered); + + let decoded: Request = serde_json::from_value(serde_json::json!({ + "operation": "remove", + "instance_id": "instance", + "vm_id": "vm", + "nic_index": 0, + "filtered": false, + })) + .unwrap(); + let Request::Remove { filtered, .. } = decoded else { + panic!("wrong variant"); + }; + assert!(!filtered); + } + + /// A binding outlives the interface it was bound to, and TAP names are a + /// deterministic hash of the VM identity, so the same name comes back. + /// Removing an interface therefore clears whatever binding is there, and + /// only insists when the caller is about to create a replacement. + #[test] + fn binding_cleanup_insists_only_when_a_replacement_follows() { + assert_eq!(binding_cleanup(true), BindingCleanup::Required); + assert_eq!(binding_cleanup(false), BindingCleanup::BestEffort); + } + + #[test] + fn queue_counts_normalize_to_at_least_one_and_stay_bounded() { + assert_eq!(validate_queues(0).unwrap(), 1); + assert_eq!(validate_queues(1).unwrap(), 1); + assert_eq!(validate_queues(MAX_QUEUES).unwrap(), MAX_QUEUES); + assert!(validate_queues(MAX_QUEUES + 1).is_err()); + } + + #[test] + fn queue_count_travels_with_the_prepare_request() { + let request = Request::PrepareBridge(PrepareBridgeRequest { + identity: identity("instance", "vm", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filter: String::new(), + parameters: BTreeMap::new(), + queues: 4, + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["queues"], 4); + assert_eq!(value["filter"], ""); + + // Older VMMs omit the field entirely; that must stay single-queue. + let decoded: Request = serde_json::from_value(serde_json::json!({ + "operation": "prepare_bridge", + "instance_id": "instance", + "vm_id": "vm", + "nic_index": 0, + "bridge": "br0", + "mac": "02:00:00:00:00:01", + "qemu_uid": 1000, + "filter": "clean-traffic", + })) + .unwrap(); + let Request::PrepareBridge(decoded) = decoded else { + panic!("wrong variant"); + }; + assert_eq!(decoded.queues, 0); + assert_eq!(validate_queues(decoded.queues).unwrap(), 1); + } + #[test] fn macvtap_prepare_has_a_dedicated_operation() { let request = Request::PrepareMacvtap(PrepareMacvtapRequest { @@ -727,6 +951,7 @@ mod tests { mac: "02:00:00:00:00:01".into(), qemu_uid: 1000, mode: "private".into(), + queues: 0, }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_macvtap"); @@ -744,6 +969,7 @@ mod tests { qemu_uid: 1000, filter: "clean-traffic".into(), parameters: BTreeMap::new(), + queues: 0, }); let value = serde_json::to_value(request).unwrap(); assert_eq!(value["operation"], "prepare_bridge"); @@ -752,8 +978,14 @@ mod tests { assert!(value.get("identity").is_none()); } + /// Connecting and closing without sending is how the VMM checks that netd + /// is alive, because a netd that died leaves its socket behind. It has to + /// be handled promptly, and quietly: the VMM does it once per status query + /// that mentions a stopped VM, and netd's accept loop is serialized, so + /// treating a probe as a failed request would both fill the log and put + /// noise in front of real work. #[tokio::test] - async fn disconnected_client_is_confined_to_one_connection() { + async fn a_connection_that_sends_nothing_is_a_liveness_probe() { let (mut server, client) = UnixStream::pair().unwrap(); drop(client); let result = timeout( @@ -762,10 +994,37 @@ mod tests { ) .await; assert!(result.is_ok(), "disconnected peer blocked the handler"); - // Either the EOF is reported while reading or the response write sees - // EPIPE. In both cases serve() logs this per-connection error and keeps - // accepting clients. - assert!(result.unwrap().is_err()); + assert!(result.unwrap().is_ok(), "a probe is not a failed request"); + } + + /// Only an empty connection is a probe. A peer that does send something, + /// and sends nonsense, is still a request -- and still gets an answer it + /// can read, which is how a VMM newer than its netd learns to say so. + #[tokio::test] + async fn a_request_that_cannot_be_understood_still_gets_an_answer() { + let (mut server, client) = UnixStream::pair().unwrap(); + drop(client); + assert!(read_request(&mut server).await.unwrap().is_none()); + + let (mut server, mut client) = UnixStream::pair().unwrap(); + // An operation only a newer VMM knows about. + client + .write_all(br#"{"operation":"prepare_something_new"}"#) + .await + .unwrap(); + client.shutdown().await.unwrap(); + serve_connection(&NetdConfig::default(), &mut server) + .await + .unwrap(); + + let mut reply = Vec::new(); + client.read_to_end(&mut reply).await.unwrap(); + let reply: serde_json::Value = serde_json::from_slice(&reply).unwrap(); + assert_eq!(reply["ok"], false); + assert!( + reply["error"].as_str().unwrap().contains("unknown variant"), + "{reply}" + ); } #[test] diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index dd7a23d79..c53bb5744 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -3,10 +3,10 @@ // SPDX-License-Identifier: Apache-2.0 use crate::app::{ - make_sys_config, resolved_networks, simulator_config_for_manifest, sync_tee_simulator_config, - Image, VmConfig, VmWorkDir, + clamp_queues_without_netd, make_sys_config, needs_netd_interface, resolved_networks, + simulator_config_for_manifest, sync_tee_simulator_config, Image, VmConfig, VmWorkDir, }; -use crate::config::{Config, NetworkFilterMode, NetworkingMode}; +use crate::config::Config; use crate::main_service; use anyhow::{Context, Result}; use fs_err as fs; @@ -279,18 +279,33 @@ Compose file content (first 200 chars): gateway_enabled: app_compose.gateway_enabled(), }; + // One-shot has no netd lifecycle, so a bridge NIC that only wanted the + // vCPU-scaled default drops to a single queue here exactly as it would on a + // server without netd. Anything still needing an interface was asked for + // explicitly, and is refused rather than silently downgraded. + let requested = if manifest.networks.is_empty() { + vec![config.cvm.networking.clone()] + } else { + manifest.networks.clone() + }; + let mut runtime_networks = resolved_networks(&manifest, &config.cvm); + let clamped = clamp_queues_without_netd(&requested, &mut runtime_networks, &config.cvm, false); + if clamped > 0 { + tracing::warn!( + "one-shot execution has no netd, so {clamped} bridge interface(s) fall back to a \ + single queue pair; run the VMM server to let queue pairs scale with vCPUs" + ); + } if !dry_run - && config.cvm.network_filter.mode == NetworkFilterMode::Libvirt - && resolved_networks(&manifest, &config.cvm) + && runtime_networks .iter() - .any(|network| network.mode == NetworkingMode::Bridge) + .any(|network| needs_netd_interface(network, &config.cvm)) { anyhow::bail!( - "one-shot execution does not manage libvirt-filtered TAP lifecycle; run the VMM server directly or use --dry-run" + "one-shot execution does not manage netd interface lifecycle; run the VMM server directly or use --dry-run" ); } - let runtime_networks = resolved_networks(&manifest, &config.cvm); let process_configs = vm_builder_config .config_qemu(&workdir_path, &config.cvm, &gpus, &runtime_networks) .context("Failed to build QEMU configuration")?; diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 1a533434f..6def0bdb9 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -919,8 +919,18 @@ def create_vm(self, args) -> None: params["kms_urls"] = args.kms_url if args.gateway_url: params["gateway_urls"] = args.gateway_url - if args.net: - params["networking"] = {"mode": args.net} + # "auto" is what a fresh deployment already does, so it only means + # something to `update`, where it clears a pinned count. + net_queues = None if args.net_queues == "auto" else args.net_queues + if args.net or args.net_vhost is not None or net_queues: + networking = {} + if args.net: + networking["mode"] = args.net + if args.net_vhost is not None: + networking["vhost"] = args.net_vhost + if net_queues: + networking["queues"] = net_queues + params["networking"] = networking app_id = args.app_id or self.calc_app_id(compose_content) print(f"App ID: {app_id}") @@ -1030,6 +1040,10 @@ def update_vm( no_gpus: bool = False, kms_urls: Optional[List[str]] = None, no_tee: Optional[bool] = None, + net: Optional[str] = None, + net_vhost: Optional[bool] = None, + net_vhost_inherit: bool = False, + net_queues: Optional[Union[int, str]] = None, ) -> None: """Update multiple aspects of a VM in one command.""" # Validate: --env-file requires --kms-url @@ -1153,6 +1167,56 @@ def update_vm( app_compose, indent=4, ensure_ascii=False ) + if net or net_vhost is not None or net_vhost_inherit or net_queues: + # The RPC replaces the whole NIC list, so merge into what the VM + # already has rather than silently dropping its other interfaces or + # un-pinning a bridge it was deployed with. + if vm_info_response is None: + vm_info_response = self.rpc_call("GetInfo", {"id": vm_id}) + if not vm_info_response.get("found", False): + raise Exception(f"VM with ID {vm_id} not found") + configuration = vm_info_response["info"].get("configuration") or {} + current = configuration.get("networks") or [] + if not current and configuration.get("networking"): + current = [configuration["networking"]] + if len(current) > 1: + raise Exception( + "this VM has multiple network interfaces; edit them through the " + "web UI or the UpgradeApp API rather than these flags" + ) + # Only the fields the deployment RPC accepts back travel with the + # update. macvtap_mode is node-controlled and can never be changed, + # so resending it can only fail if the node changed meanwhile. An + # empty mode is meaningful: it says the VM never named a backend + # and still follows the node's. + source = current[0] if current else {} + networking = { + key: source[key] + for key in ("mode", "bridge_name", "parent", "vhost", "queues") + if source.get(key) not in (None, "") + } + if net: + networking["mode"] = net + # A field belongs to the mode that owns it. Carrying a bridge into + # a macvtap request, or a parent into a bridge one, asks the server + # about a field the caller never typed and has no flag to clear. + mode = networking.get("mode", "") + if mode and mode != "bridge": + networking.pop("bridge_name", None) + if mode and mode != "macvtap": + networking.pop("parent", None) + if net_vhost is not None: + networking["vhost"] = net_vhost + elif net_vhost_inherit: + networking.pop("vhost", None) + if net_queues == "auto": + networking.pop("queues", None) + elif net_queues: + networking["queues"] = net_queues + upgrade_params["update_networking"] = True + upgrade_params["networks"] = [networking] + updates.append(f"networking ({networking})") + if user_config: upgrade_params["user_config"] = user_config updates.append("user config") @@ -1247,6 +1311,25 @@ def show_info(self, vm_id: str, json_output: bool = False) -> None: if info.get("shutdown_progress"): print(f"Shutdown: {info['shutdown_progress']}") + interfaces = info.get("interfaces") or [] + if interfaces: + print("\nNetwork Interfaces:") + for iface in interfaces: + parts = [ + f"{iface.get('netdev_id') or '-':<6}", + f"{iface.get('mode') or '-'}/{iface.get('backend') or '-'}", + iface.get("mac") or "-", + ] + if iface.get("bridge_name"): + parts.append(f"bridge={iface['bridge_name']}") + parts.append("vhost=" + ("on" if iface.get("vhost") else "off")) + parts.append(f"queues={iface.get('queues', 1)}") + print(" " + " ".join(parts)) + # A stopped VM has no interfaces to describe, so these are what its + # next launch would build -- which can differ from its last one. + if (info.get("status") or "") not in ("running", "stopping"): + print(" (not running; shown as its next launch would build them)") + events = info.get("events", []) if events: print("\nRecent Events:") @@ -1536,6 +1619,26 @@ def save_whitelist(whitelist: List[str]) -> None: json.dump({"trusted_signers": whitelist}, f, indent=2) +def queue_count(value: str) -> Union[int, str]: + """A queue pair count the node could act on, or "auto" to stop pinning one. + + Zero would otherwise reach the wire as "unset" and be answered with the + default, and a negative one as a decoding error naming a column offset -- + neither of which tells the caller what they asked for was impossible. + """ + if value == "auto": + return "auto" + try: + count = int(value) + except ValueError: + raise argparse.ArgumentTypeError(f"'{value}' is not a whole number or 'auto'") + if count < 1: + raise argparse.ArgumentTypeError( + f"queue pairs must be at least 1, or 'auto' to follow the vCPU count; got {count}" + ) + return count + + def main(): """Parse arguments and dispatch to the appropriate command handler.""" parser = argparse.ArgumentParser(description="dstack-vmm CLI - Manage VMs") @@ -1831,9 +1934,31 @@ def _patched_format_help(): ) deploy_parser.add_argument( "--net", - choices=["bridge", "user"], + choices=["bridge", "user", "macvtap"], help="Networking mode (default: use global config)", ) + net_vhost = deploy_parser.add_mutually_exclusive_group() + net_vhost.add_argument( + "--net-vhost", + dest="net_vhost", + action="store_true", + default=None, + help="Use the host kernel vhost-net data plane (default: use global config)", + ) + net_vhost.add_argument( + "--net-no-vhost", + dest="net_vhost", + action="store_false", + help="Keep packet processing in the QEMU main loop", + ) + deploy_parser.add_argument( + "--net-queues", + type=queue_count, + metavar="N", + help="virtio-net queue pairs, bounded by the node's max_net_queues. " + "Without --net, the node's own networking mode is kept " + "(default: use global config)", + ) # Images command lsimage_parser = subparsers.add_parser("lsimage", help="List available images") @@ -1934,6 +2059,38 @@ def _patched_format_help(): "--env-file", help="File with environment variables to encrypt" ) update_parser.add_argument("--user-config", help="Path to user config file") + update_parser.add_argument( + "--net", + choices=["bridge", "user", "macvtap"], + help="Networking mode (applies from the next boot)", + ) + update_net_vhost = update_parser.add_mutually_exclusive_group() + update_net_vhost.add_argument( + "--net-vhost", + dest="net_vhost", + action="store_true", + default=None, + help="Use the host kernel vhost-net data plane", + ) + update_net_vhost.add_argument( + "--net-no-vhost", + dest="net_vhost", + action="store_false", + help="Keep packet processing in the QEMU main loop", + ) + update_net_vhost.add_argument( + "--net-vhost-default", + dest="net_vhost_inherit", + action="store_true", + help="Stop pinning vhost and follow the node default again", + ) + update_parser.add_argument( + "--net-queues", + type=queue_count, + metavar="N", + help="virtio-net queue pairs, bounded by the node's max_net_queues. " + "Use 'auto' to stop pinning a count and follow the vCPU count again", + ) # Port mapping options (mutually exclusive with --no-ports) port_group = update_parser.add_mutually_exclusive_group() port_group.add_argument( @@ -2077,6 +2234,10 @@ def _patched_format_help(): no_gpus=args.no_gpus if hasattr(args, "no_gpus") else False, kms_urls=args.kms_url, no_tee=args.no_tee, + net=args.net, + net_vhost=args.net_vhost, + net_vhost_inherit=getattr(args, "net_vhost_inherit", False), + net_queues=args.net_queues, ) elif args.command == "kms": if not args.kms_action: diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index 82653a62d..29a6a8679 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -22,6 +22,7 @@ const CreateVmDialogComponent = { portMappingEnabled: { type: Boolean, required: true }, networkingModes: { type: Array, required: true }, defaultBridge: { type: String, default: '' }, + maxNetQueues: { type: Number, default: 0 }, defaultNetworkingLabel: { type: String, required: true }, }, emits: ['close', 'submit', 'load-compose'], @@ -161,6 +162,7 @@ const CreateVmDialogComponent = {
{{ defaultNetworkingLabel }}
+ + + + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} Guest IP is assigned by host DHCP on that bridge and reported after boot.
- + diff --git a/dstack/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts index 5dc9568b1..ddca696fc 100644 --- a/dstack/vmm/ui/src/components/UpdateVmDialog.ts +++ b/dstack/vmm/ui/src/components/UpdateVmDialog.ts @@ -21,6 +21,7 @@ const UpdateVmDialogComponent = { portMappingEnabled: { type: Boolean, required: true }, networkingModes: { type: Array, required: true }, defaultBridge: { type: String, default: '' }, + maxNetQueues: { type: Number, default: 0 }, defaultNetworkingLabel: { type: String, required: true }, kmsEnabled: { type: Boolean, required: true }, composeHashPreview: { type: String, required: true }, @@ -150,6 +151,7 @@ const UpdateVmDialogComponent = {
{{ defaultNetworkingLabel }}
+ + + + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} Guest IP is assigned by host DHCP on that bridge and reported after boot.
- + diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index 1abb5ab75..1c963d794 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -111,6 +111,12 @@ type PortFormEntry = { type NetworkFormEntry = { mode: string; bridge_name?: string; + /** Pinned at deployment for macvtap NICs; carried through edits unchanged. */ + parent?: string; + /** '' inherits the node default, otherwise 'on' or 'off'. */ + vhost?: string; + /** '' lets the queue count follow the vCPU count. */ + queues?: string; }; type VmFormState = { @@ -351,6 +357,7 @@ fi return Array.from(new Set(fallback)); }); const defaultBridge = computed(() => config.value.networking?.default_bridge || ''); + const maxNetQueues = computed(() => config.value.networking?.max_queues || 0); const defaultNetworkingLabel = computed(() => { const mode = config.value.networking?.default_mode || ''; if (mode === 'bridge') { @@ -450,16 +457,42 @@ fi return configured.map((network) => ({ mode: network.mode || '', bridge_name: network.bridge_name || '', + parent: network.parent || '', + vhost: network.vhost === null || network.vhost === undefined ? '' : (network.vhost ? 'on' : 'off'), + queues: network.queues ? String(network.queues) : '', })); }; const normalizeNetworks = (networks: NetworkFormEntry[] = []): VmmTypes.INetworkingConfig[] => networks - .map((network) => ({ - mode: (network.mode || '').trim(), - bridge_name: network.mode === 'bridge' ? (network.bridge_name || '').trim() : '', - })) - .filter((network) => network.mode.length > 0); + .map((network) => { + // Leave vhost and queues unset unless the operator picked something, so + // the node keeps owning them and can still change them later. + const entry: VmmTypes.INetworkingConfig = { + mode: (network.mode || '').trim(), + bridge_name: network.mode === 'bridge' ? (network.bridge_name || '').trim() : '', + parent: network.mode === 'macvtap' ? (network.parent || '').trim() : '', + }; + // The tuning controls are hidden for user mode, so sending values the + // operator cannot see would fail the deploy with nothing to fix. + if (network.mode !== 'user') { + if (network.vhost === 'on' || network.vhost === 'off') { + entry.vhost = network.vhost === 'on'; + } + const queues = Number.parseInt(network.queues || '', 10); + if (Number.isFinite(queues) && queues > 0) { + entry.queues = queues; + } + } + return entry; + }); + // Nothing is filtered out. A mode-less entry is the "keep the node's + // backend, change only the data plane" override the RPC accepts, and is + // what a VM deployed that way reports back; dropping it would delete a + // NIC and renumber the ones after it, which changes their MAC addresses. + // Adding a network always picks a mode, so the only way to reach an + // entry the RPC refuses is to empty a loaded one, and that earns an + // error rather than silence. function networkModeLabel(mode?: string | null) { if (!mode) { @@ -1833,6 +1866,7 @@ type CreateVmPayloadSource = { config, networkingModes, defaultBridge, + maxNetQueues, defaultNetworkingLabel, composeHashPreview, updateComposeHashPreview, diff --git a/dstack/vmm/ui/src/styles/main.css b/dstack/vmm/ui/src/styles/main.css index 7a7b0ee72..f778428b9 100644 --- a/dstack/vmm/ui/src/styles/main.css +++ b/dstack/vmm/ui/src/styles/main.css @@ -620,7 +620,7 @@ h1, h2, h3, h4, h5, h6 { .runtime-network-item { display: grid; - grid-template-columns: 1.1fr 0.6fr 0.8fr 1.4fr; + grid-template-columns: 1.1fr 0.6fr 0.8fr 1.4fr 0.7fr 0.7fr; gap: 12px; overflow-wrap: anywhere; } @@ -1422,11 +1422,24 @@ h1, h2, h3, h4, h5, h6 { .network-config-row { display: grid; - grid-template-columns: 160px minmax(180px, 1fr) 100px; + grid-template-columns: 160px minmax(180px, 1fr) auto 100px; gap: 12px; align-items: center; } +.network-config-tuning { + display: flex; + gap: 8px; +} + +.network-config-tuning select { + width: 140px; +} + +.network-config-tuning input { + width: 190px; +} + .network-config-placeholder { min-height: 1px; } diff --git a/dstack/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html index 139155032..b68e8635b 100644 --- a/dstack/vmm/ui/src/templates/app.html +++ b/dstack/vmm/ui/src/templates/app.html @@ -80,6 +80,7 @@

dstack-vmm

:port-mapping-enabled="config.portMappingEnabled" :networking-modes="networkingModes" :default-bridge="defaultBridge" + :max-net-queues="maxNetQueues" :default-networking-label="defaultNetworkingLabel" @close="showCreateDialog = false" @submit="createVm" @@ -95,6 +96,7 @@

dstack-vmm

:port-mapping-enabled="config.portMappingEnabled" :networking-modes="networkingModes" :default-bridge="defaultBridge" + :max-net-queues="maxNetQueues" :default-networking-label="defaultNetworkingLabel" :kms-enabled="kmsEnabled(updateDialog.vm || {})" :compose-hash-preview="updateComposeHashPreview" @@ -345,11 +347,16 @@

Port Mappings

VMM Network Interfaces

+ + Not running; shown as the next launch would build them. +
{{ networkModeLabel(iface.mode) }} / {{ iface.backend || '-' }} {{ iface.netdev_id || '-' }} {{ iface.bridge_name || '-' }} {{ iface.mac || '-' }} + vhost: {{ iface.vhost ? 'on' : 'off' }} + queues: {{ iface.queues || 1 }}
diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 26f221866..b038c6141 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -54,6 +54,15 @@ allowed_network_modes = ["user", "bridge"] # Empty allowlists mean callers can only use the node networking defaults. allowed_bridges = [] allowed_macvtap_parents = [] +# Largest virtio-net queue pair count a deployment request may ask for. Queue +# pairs otherwise default to the VM's vCPU count, capped at 16; raising this +# above 16 widens what a caller may request without moving that default, and +# lowering it below 16 lowers the default too. Bridge mode needs netd for +# anything above 1, because qemu-bridge-helper cannot create a multiqueue TAP. +# Without netd an unfiltered bridge NIC that took the default drops to one +# queue; one that asked for a count keeps it and fails to launch instead, so +# the caller learns their request was not met. +max_net_queues = 16 use_mrconfigid = true # QEMU flags @@ -62,6 +71,10 @@ use_mrconfigid = true #qemu_version = "" qemu_pci_hole64_size = 0 qemu_hotplug_off = false +# Path to qemu-bridge-helper, needed by vhost bridge networking because QEMU's +# `tap` netdev, unlike its `bridge` netdev, has no compiled-in default. Empty +# probes the known distribution locations. +#qemu_bridge_helper = "/usr/lib/qemu/qemu-bridge-helper" # TDX attestation/hash scheme policy: # - "legacy": digest.txt + legacy verifier # - "lite": digest.txt + measurement.tdx.cbor + no-QEMU verifier @@ -111,6 +124,12 @@ product_name = "dstack" [cvm.networking] mode = "user" +# Kernel vhost-net data plane. With it off, QEMU drains every packet on its +# single main loop thread, so a CVM cannot exceed one core's worth of packet +# processing no matter how many vCPUs it has. The user-mode backend has no +# vhost support and ignores this. Individual VMs may override it. +vhost = true + # for mode = "user" net = "10.0.2.0/24" dhcp_start = "10.0.2.10" @@ -120,15 +139,17 @@ restrict = false # bridge = "virbr0" # Optional filtering for bridge interfaces only. It does not apply to macvtap. -# "none" preserves the existing QEMU bridge-helper behavior and has no -# netd/libvirt dependency. +# "none" installs no nwfilter binding. It does not by itself remove the netd +# dependency: netd also builds the multiqueue TAP that qemu-bridge-helper +# cannot create, so a bridge node without netd is limited to one queue pair. [cvm.network_filter] mode = "none" filter = "clean-traffic" parameters = {} -# Shared privileged networking service. Only used when network_filter.mode is -# "libvirt". Socket filesystem permissions authorize clients. +# Shared privileged networking service. Used for macvtap NICs, for libvirt +# filtering, and for multiqueue bridge NICs. Socket filesystem permissions +# authorize clients. [netd] socket = "/run/dstack/netd.sock" # Applied when netd creates the socket itself. A systemd socket unit controls