From 56fc5db5c1f6f95bdb83fc8da1a2163c09a01ea8 Mon Sep 17 00:00:00 2001 From: Shiju Date: Sun, 8 Nov 2026 23:45:19 +0530 Subject: [PATCH 1/3] fix(vm): create missing fd links during guest init Signed-off-by: Shiju --- crates/openshell-driver-vm/README.md | 6 ++ .../scripts/openshell-vm-sandbox-init.sh | 25 ++++++ crates/openshell-driver-vm/src/rootfs.rs | 87 +++++++++++++++++++ 3 files changed, 118 insertions(+) diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index cb9433d08e..46595bf144 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -14,6 +14,12 @@ binaries. The driver embeds libkrun, libkrunfw, the guest OCI unpacker, the portable guest sandbox, and the custom kernel runtime. Each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir. +Guest init creates missing `/dev/fd` and standard-stream links after mounting +procfs and devtmpfs, before using Bash process substitution. Existing links to +the expected procfs paths, or entries that resolve to the same files, stay +intact. Conflicting entries and unrelated dangling links fail boot with a +diagnostic rather than being replaced. + ## How it fits together Caller driver config is disabled by default, including upload workflows diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 37950675b0..97a99e1078 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -31,6 +31,31 @@ mount_initial_fs() { mount -t tmpfs tmpfs /tmp 2>/dev/null || true mount -t tmpfs tmpfs /run 2>/dev/null || true mount -t devtmpfs devtmpfs /dev 2>/dev/null || true + setup_fd_links /dev +} + +setup_fd_links() { + local dev_dir="$1" + local name target + for name in fd stdin stdout stderr; do + case "$name" in + fd) target=/proc/self/fd ;; + stdin) target=/proc/self/fd/0 ;; + stdout) target=/proc/self/fd/1 ;; + stderr) target=/proc/self/fd/2 ;; + esac + if [[ -L "$dev_dir/$name" ]] && [[ "$(readlink "$dev_dir/$name")" == "$target" ]]; then + continue + fi + if [[ -e "$dev_dir/$name" || -L "$dev_dir/$name" ]]; then + if [[ ! "$dev_dir/$name" -ef "$target" ]]; then + printf 'guest init: %s must refer to %s\n' "$dev_dir/$name" "$target" >&2 + return 1 + fi + else + ln -s "$target" "$dev_dir/$name" + fi + done } bind_mount_into_newroot() { diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index a292e81994..67846c10ac 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -1357,6 +1357,93 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn guest_init_creates_fd_links_and_preserves_existing_aliases() { + use std::os::unix::fs::MetadataExt as _; + + let dev = tempfile::tempdir().expect("temporary guest dev"); + let run = || run_guest_setup_fd_links(dev.path()); + let output = run(); + assert!(output.status.success(), "{output:?}"); + let aliases = [ + ("fd", "/proc/self/fd"), + ("stdin", "/proc/self/fd/0"), + ("stdout", "/proc/self/fd/1"), + ("stderr", "/proc/self/fd/2"), + ]; + let mut inodes = Vec::new(); + for (name, target) in aliases { + let path = dev.path().join(name); + assert_eq!(fs::read_link(&path).expect("read alias"), Path::new(target)); + inodes.push(fs::symlink_metadata(path).expect("stat alias").ino()); + } + let output = run(); + assert!(output.status.success(), "{output:?}"); + for ((name, _), inode) in aliases.into_iter().zip(inodes) { + assert_eq!( + fs::symlink_metadata(dev.path().join(name)) + .expect("stat retained alias") + .ino(), + inode, + "existing {name} alias stays intact" + ); + } + #[cfg(target_os = "linux")] + { + let stdin = dev.path().join("stdin"); + fs::remove_file(&stdin).expect("replace fixture alias"); + std::os::unix::fs::symlink("fd/0", &stdin).expect("equivalent relative alias"); + let output = run(); + assert!(output.status.success(), "{output:?}"); + assert_eq!( + fs::read_link(stdin).expect("retained relative alias"), + Path::new("fd/0") + ); + } + } + + #[test] + fn guest_init_rejects_conflicting_fd_entries_without_replacing_them() { + let dev = tempfile::tempdir().expect("temporary guest dev"); + let fd = dev.path().join("fd"); + fs::write(&fd, b"keep this entry").expect("conflicting file"); + let output = run_guest_setup_fd_links(dev.path()); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("must refer to /proc/self/fd")); + assert_eq!(fs::read(&fd).expect("retained file"), b"keep this entry"); + + fs::remove_file(&fd).expect("remove fixture file"); + std::os::unix::fs::symlink("missing-target", &fd).expect("dangling link"); + let output = run_guest_setup_fd_links(dev.path()); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("must refer to /proc/self/fd")); + assert_eq!( + fs::read_link(fd).expect("retained link"), + Path::new("missing-target") + ); + } + + fn run_guest_setup_fd_links(dev: &Path) -> std::process::Output { + let script = include_str!("../scripts/openshell-vm-sandbox-init.sh"); + let start = script + .find("setup_fd_links() {\n") + .expect("fd setup function"); + let tail = &script[start..]; + let end = tail.find("\n}\n").expect("fd setup function end") + "\n}\n".len(); + Command::new("bash") + .args([ + "-c", + &format!( + "set -euo pipefail\n{}\nsetup_fd_links \"$1\"\n", + &tail[..end] + ), + "test", + ]) + .arg(dev) + .output() + .expect("run shipped fd setup") + } + /// Runs the shipped guest init's `configure_hosts` against `root`. /// /// The function is cut out of the real script so the test exercises the From 32118a30729c91c9e790519dcbb96bdeff4b05f1 Mon Sep 17 00:00:00 2001 From: Shiju Date: Sun, 8 Nov 2026 23:46:10 +0530 Subject: [PATCH 2/3] feat(vm): validate launch requirements against runtime capabilities Signed-off-by: Shiju --- crates/openshell-driver-vm/README.md | 35 +++ crates/openshell-driver-vm/src/driver.rs | 140 ++++++--- crates/openshell-driver-vm/src/lifecycle.rs | 84 +++-- crates/openshell-driver-vm/src/runtime.rs | 320 ++++++++++++++++---- docs/how-it-works/sandboxes/runtimes.mdx | 6 + 5 files changed, 463 insertions(+), 122 deletions(-) diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 46595bf144..e9394df901 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -20,6 +20,41 @@ the expected procfs paths, or entries that resolve to the same files, stay intact. Conflicting entries and unrelated dangling links fail boot with a diagnostic rather than being replaced. +## Lifecycle extension requirements + +Lifecycle extensions declare requirements through `LaunchPlan::require_backend` +and `require_backend_feature`. An explicit backend stays selected; incompatible +requirements fail before the workload launcher starts. Without an explicit +requirement, the driver chooses a backend that supports the requested features. + +These are VM-driver-local backend capabilities. The driver resolves the final +plan after operator-installed extensions contribute their requirements, then +checks it against the selected backend. The gateway's `GetCapabilities` +snapshot describes the compute-driver protocol and supported resource request +forms; it does not describe this per-launch VM plan. + +| Feature | libkrun | QEMU | +|---|---|---| +| Guest init drop-ins | Supported | Supported | +| External kernel image | Unsupported | Supported for CPU-only and GPU VMs | +| PCI passthrough | Unsupported | Requires an assigned GPU device | +| Host directory mounts | Supported through virtiofs | Unsupported | + +For example, an operator-installed extension can set `plan.kernel_image` and +require `BackendFeature::ExternalKernelImage` for a CPU-only sandbox. The +resolver selects QEMU with a vsock CID and ordinary CPU sizing, without +allocating a GPU. Requiring libkrun for that same plan fails with an +external-kernel diagnostic. Kernel paths remain extension-owned; sandbox +requests cannot supply them. + +The current QEMU launcher requires a Linux x86_64 host with KVM. Other host +targets fail preflight before the workload launcher starts. External kernels are checked for a +64-bit little-endian x86_64 ELF header or an x86_64 Linux bzImage header. The +VMM validates the complete image. Kernel security and boot-driver requirements +still apply, and authenticated runtime admission confirms enforcement before +workload code runs. Both workload and image-preparation launchers use the same +kernel preflight entry point; preparation retains its libkrun bootstrap backend. + ## How it fits together Caller driver config is disabled by default, including upload workflows diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index f7946c34c2..2938a22245 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1994,9 +1994,11 @@ impl VmDriver { } if let Err(err) = Self::validate_launch_plan_backend(is_gpu, &plan).and_then(|()| { - // Cannot fail today: `validate_sandbox` already ran and non-GPU - // QEMU is blocked above. Keeps virtiofs mounts off QEMU once that - // guard is lifted. + plan.backend + .validate_host() + .map_err(Status::failed_precondition)?; + // Extensions can select QEMU after request validation, so check + // the selected backend's mount support again before launch. validate_vm_driver_mounts( &driver_config.mounts, &self.config, @@ -3003,13 +3005,30 @@ impl VmDriver { plan.require_backend_feature(BackendFeature::GuestInitDropins); } - if plan.required_backends.contains(&VmBackend::Qemu) - || plan.backend == VmBackend::Qemu - || plan + if let Some(required) = plan.required_backends.first() { + if plan + .required_backends + .iter() + .any(|backend| backend != required) + { + return Err(Status::failed_precondition( + "VM lifecycle extensions requested conflicting VM backends", + )); + } + plan.backend = required.clone(); + } else if plan + .required_backend_features + .iter() + .any(|feature| !feature.is_supported_by(&plan.backend)) + && plan .required_backend_features .iter() - .any(|feature| feature.requires_qemu()) + .all(|feature| feature.is_supported_by(&VmBackend::Qemu)) { + plan.backend = VmBackend::Qemu; + } + + if plan.backend == VmBackend::Qemu { self.configure_qemu_launch_plan(sandbox_id, is_gpu, gpu_bdf, plan)?; } @@ -3043,33 +3062,21 @@ impl VmDriver { #[allow(clippy::result_large_err)] fn validate_launch_plan_backend(is_gpu: bool, plan: &LaunchPlan) -> Result<(), Status> { - // NOTE: this guard exists because the non-GPU QEMU launch path - // (PCI device transport, VFIO root port wiring) has not landed - // yet. Until then, even though the resolver will happily promote - // a plan to QEMU when an extension requires `PciPassthrough` or - // `ExternalKernelImage`, the launch itself is blocked here so we - // don't spawn a QEMU instance with no concrete device backing. - // Remove this guard once the non-GPU QEMU launch path supports - // emitting `pcie-root-port` + `vfio-pci` for arbitrary device - // descriptors. - if plan.backend == VmBackend::Qemu && !is_gpu { - let offending_feature = plan - .required_backend_features - .iter() - .find(|feature| feature.requires_qemu()) - .map_or("(explicit QEMU backend requirement)", |feature| { - feature.as_str() - }); - return Err(Status::failed_precondition(format!( - "vm lifecycle extension required '{offending_feature}', which resolves to the QEMU backend, \ - but non-GPU QEMU launch is not yet supported (pending PCI device transport)" - ))); - } if plan.backend != VmBackend::Qemu && is_gpu { return Err(Status::failed_precondition( "GPU sandbox launch requires the QEMU backend", )); } + if plan.gpu_bdf.is_some() && !is_gpu { + return Err(Status::failed_precondition( + "GPU passthrough requires a GPU sandbox request", + )); + } + if is_gpu && plan.gpu_bdf.is_none() { + return Err(Status::failed_precondition( + "GPU sandbox launch is missing its assigned GPU device", + )); + } if plan.required_backends.contains(&VmBackend::Libkrun) && plan.required_backends.contains(&VmBackend::Qemu) { @@ -3089,29 +3096,30 @@ impl VmDriver { "VM lifecycle extension requires the QEMU backend", )); } - if plan.backend != VmBackend::Qemu - && let Some(feature) = plan - .required_backend_features - .iter() - .find(|feature| feature.requires_qemu()) + if let Some(feature) = plan + .required_backend_features + .iter() + .find(|feature| !feature.is_supported_by(&plan.backend)) { return Err(Status::failed_precondition(format!( - "VM backend feature '{}' requires a VM backend with PCI-style launch support", + "VM backend {:?} does not support feature '{}'", + plan.backend, feature.as_str() ))); } - if plan.kernel_image.is_some() && plan.backend != VmBackend::Qemu { + if plan + .required_backend_features + .contains(&BackendFeature::PciPassthrough) + && plan.gpu_bdf.is_none() + { return Err(Status::failed_precondition( - "selected kernel image requires a VM backend that supports external kernel images", + "PCI passthrough requires an assigned GPU device; arbitrary PCI devices are not supported", )); } - if let Some(kernel_image) = &plan.kernel_image - && !kernel_image.is_file() - { - return Err(Status::failed_precondition(format!( - "selected kernel image does not exist: {}", - kernel_image.display() - ))); + if let Some(kernel_image) = &plan.kernel_image { + plan.backend + .validate_kernel_image(kernel_image) + .map_err(Status::failed_precondition)?; } Ok(()) } @@ -13693,10 +13701,28 @@ while :; do sleep 0.05; done"#, let base = unique_temp_dir(); std::fs::create_dir_all(&base).unwrap(); let kernel = base.join("vmlinux"); - std::fs::write(&kernel, b"kernel").unwrap(); - plan.backend = VmBackend::Qemu; + let mut header = [0u8; 64]; + header[..6].copy_from_slice(b"\x7fELF\x02\x01"); + header[18] = 62; // EM_X86_64; a header fixture, not a bootable guest. + std::fs::write(&kernel, header).unwrap(); plan.kernel_image = Some(kernel); - VmDriver::validate_launch_plan_backend(true, &plan).expect("existing kernel is accepted"); + let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + plan.require_backend(VmBackend::Libkrun); + driver + .resolve_launch_plan_backend("kernel-check", false, None, &mut plan) + .unwrap(); + assert_eq!(plan.backend, VmBackend::Libkrun); + assert!(VmDriver::validate_launch_plan_backend(false, &plan).is_err()); + + plan.required_backends.clear(); + driver + .resolve_launch_plan_backend("kernel-check", false, None, &mut plan) + .unwrap(); + assert_eq!(plan.backend, VmBackend::Qemu); + assert!(plan.gpu_bdf.is_none()); + assert!(plan.vsock_cid.is_some()); + VmDriver::validate_launch_plan_backend(false, &plan) + .expect("CPU-only external kernel is accepted without PCI passthrough"); let _ = std::fs::remove_dir_all(base); } @@ -13714,6 +13740,9 @@ while :; do sleep 0.05; done"#, assert_eq!(plan.backend, VmBackend::Qemu); assert!(plan.vsock_cid.is_some()); + let error = VmDriver::validate_launch_plan_backend(false, &plan) + .expect_err("a PCI feature needs an assigned device"); + assert!(error.message().contains("assigned GPU device")); } #[test] @@ -13730,6 +13759,23 @@ while :; do sleep 0.05; done"#, assert_eq!(plan.backend, VmBackend::Qemu); assert!(plan.vsock_cid.is_some()); + VmDriver::validate_launch_plan_backend(false, &plan).expect("CPU-only QEMU is supported"); + } + + #[test] + fn conflicting_backend_requirements_fail_before_allocating_a_cid() { + let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + let mut plan = driver + .build_vm_launch_plan("conflict", false, false, None) + .unwrap(); + plan.require_backend(VmBackend::Libkrun); + plan.require_backend(VmBackend::Qemu); + let error = driver + .resolve_launch_plan_backend("conflict", false, None, &mut plan) + .expect_err("conflicting runtimes must fail"); + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("conflicting VM backends")); + assert!(plan.vsock_cid.is_none()); } #[test] diff --git a/crates/openshell-driver-vm/src/lifecycle.rs b/crates/openshell-driver-vm/src/lifecycle.rs index f92082d99e..6ce929f6a4 100644 --- a/crates/openshell-driver-vm/src/lifecycle.rs +++ b/crates/openshell-driver-vm/src/lifecycle.rs @@ -82,20 +82,9 @@ pub type LifecycleResult = Result; /// /// Extensions declare features they need (e.g. PCI passthrough or an /// external kernel image) and the VM driver resolves a concrete -/// [`VmBackend`] that can satisfy them. The mapping from feature to -/// backend lives in [`BackendFeature::requires_qemu`] for now; once a -/// third backend exists this should evolve into a per-backend capability -/// table that the resolver intersects against feature requirements. -/// -/// # Current limitations -/// -/// Until the non-GPU QEMU launch path (PCI device transport / VFIO root -/// port wiring) lands, the driver still rejects launches where the -/// resolved backend is QEMU but the sandbox has no GPU. As a result, -/// declaring [`Self::PciPassthrough`] or [`Self::ExternalKernelImage`] on -/// a non-GPU sandbox is accepted by [`LifecycleExtensionRegistry::validate`] -/// at registration time but will fail provisioning with a -/// `FailedPrecondition` status. +/// [`VmBackend`] that can satisfy them. [`Self::is_supported_by`] describes +/// each backend independently. Explicit backend requirements are preserved; +/// an unsupported feature fails before the launcher starts. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BackendFeature { /// Extension supplies its own kernel image via @@ -105,8 +94,8 @@ pub enum BackendFeature { /// [`LaunchPlan::guest_init_dropins`]. Supported by all backends. GuestInitDropins, /// Extension needs PCI device passthrough on the guest. Currently - /// QEMU-only and currently rejected for non-GPU sandboxes pending the - /// non-GPU QEMU launch path landing. + /// QEMU-only. The current device model requires an assigned GPU BDF; + /// arbitrary PCI device descriptors are not supported. PciPassthrough, } @@ -120,13 +109,15 @@ impl BackendFeature { } } - /// Returns true when satisfying this feature requires the QEMU backend - /// today. This is the simplest possible resolver and is expected to be - /// replaced with a per-backend capability table once a third backend - /// exists. + /// Whether the backend implements this feature. Device assignment and + /// kernel compatibility are validated separately for the concrete plan. #[must_use] - pub fn requires_qemu(self) -> bool { - matches!(self, Self::ExternalKernelImage | Self::PciPassthrough) + pub fn is_supported_by(self, backend: &VmBackend) -> bool { + match (backend, self) { + (VmBackend::Libkrun, Self::ExternalKernelImage | Self::PciPassthrough) => false, + (VmBackend::Libkrun | VmBackend::Qemu, Self::GuestInitDropins) + | (VmBackend::Qemu, Self::ExternalKernelImage | Self::PciPassthrough) => true, + } } } @@ -349,8 +340,11 @@ pub trait LifecycleExtension: std::fmt::Debug + Send + Sync { /// inject additional guest environment variables, and MAY return an /// error to abort the launch. Implementations MUST NOT change /// [`LaunchPlan::backend`], [`LaunchPlan::required_backends`], or - /// [`LaunchPlan::required_backend_features`]; those changes are - /// ignored by the driver once `before_launch` is reached. + /// [`LaunchPlan::required_backend_features`]; the driver has already + /// resolved and validated them and does not repeat those steps after + /// this hook. + /// The driver-owned [`LaunchPlan::gpu_bdf`] must remain unchanged; + /// the registry rejects hooks that clear or replace that assignment. /// /// If this hook performs allocations that must be released on failure /// or delete, implement @@ -604,8 +598,15 @@ impl LifecycleExtensionRegistry { plan: &mut LaunchPlan, ) -> LifecycleResult<()> { let span_status = openshell_otel::ErrorStatusGuard::current(); + let assigned_gpu = plan.gpu_bdf.clone(); for ext in self.active_for(sandbox) { ext.before_launch(sandbox, state_dir, plan).await?; + if plan.gpu_bdf != assigned_gpu { + return Err(LifecycleError::new(format!( + "extension '{}' changed the assigned GPU device in before_launch", + ext.name() + ))); + } } span_status.finish(Ok(())) } @@ -978,6 +979,41 @@ mod tests { extension.clone() } + #[derive(Debug)] + struct GpuMutatingExtension(Option); + + #[tonic::async_trait] + impl LifecycleExtension for GpuMutatingExtension { + fn name(&self) -> &'static str { + "gpu-mutator" + } + + async fn before_launch( + &self, + _sandbox: &Sandbox, + _state_dir: &Path, + plan: &mut LaunchPlan, + ) -> LifecycleResult<()> { + plan.gpu_bdf.clone_from(&self.0); + Ok(()) + } + } + + #[tokio::test] + async fn before_launch_rejects_clearing_or_replacing_the_assigned_gpu() { + for replacement in [None, Some("0000:02:00.0".to_string())] { + let registry = + LifecycleExtensionRegistry::with(vec![Arc::new(GpuMutatingExtension(replacement))]); + let mut plan = sample_plan(VmBackend::Qemu); + plan.gpu_bdf = Some("0000:01:00.0".to_string()); + let error = registry + .before_launch(&sample_sandbox(), Path::new("/tmp/state"), &mut plan) + .await + .expect_err("a requested GPU must retain its assignment after hooks"); + assert!(error.message().contains("changed the assigned GPU device")); + } + } + #[tokio::test] async fn configure_launch_runs_each_extension_in_order() { let ext_a = RecordingExtension::new("a"); diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index ad300d98cf..ad98899c8e 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -4,6 +4,7 @@ #![allow(unsafe_code)] use std::ffi::CString; +use std::io::Read as _; use std::path::{Path, PathBuf}; use std::process::{Command as StdCommand, Stdio}; use std::ptr; @@ -25,6 +26,82 @@ pub enum VmBackend { Qemu, } +impl VmBackend { + pub(crate) fn validate_host(&self) -> Result<(), String> { + match self { + Self::Libkrun => Ok(()), + Self::Qemu if cfg!(all(target_os = "linux", target_arch = "x86_64")) => Ok(()), + Self::Qemu => Err("QEMU requires a Linux x86_64 host with KVM".to_string()), + } + } + + pub(crate) fn validate_kernel_image(&self, path: &Path) -> Result<(), String> { + match self { + Self::Libkrun => Err(format!( + "libkrun does not support external kernel images: {}", + path.display() + )), + Self::Qemu => validate_qemu_kernel_image(path), + } + } +} + +/// Preflight the formats accepted by the `x86_64` QEMU launcher. The VMM still +/// validates the complete image; runtime admission confirms enforcement. +fn validate_qemu_kernel_image(path: &Path) -> Result<(), String> { + if !path.is_file() { + return Err(format!( + "selected kernel image is not a regular file: {}", + path.display() + )); + } + let file = std::fs::File::open(path).map_err(|error| { + format!( + "cannot open selected kernel image {}: {error}", + path.display() + ) + })?; + let mut header = Vec::new(); + file.take(0x238).read_to_end(&mut header).map_err(|error| { + format!( + "cannot read selected kernel image {}: {error}", + path.display() + ) + })?; + if header.starts_with(b"\x7fELF") { + // ELFCLASS64, ELFDATA2LSB and EM_X86_64. + if header.len() >= 64 + && header[4] == 2 + && header[5] == 1 + && u16::from_le_bytes([header[18], header[19]]) == 62 + { + return Ok(()); + } + return Err(format!( + "QEMU kernel must be a 64-bit little-endian x86_64 ELF image: {}", + path.display() + )); + } + // Linux/x86 boot protocol: boot_flag, HdrS, version and XLF_KERNEL_64. + // https://www.kernel.org/doc/html/latest/arch/x86/boot.html + if header.len() == 0x238 + && header[0x1fe..0x200] == [0x55, 0xaa] + && &header[0x202..0x206] == b"HdrS" + { + if u16::from_le_bytes([header[0x206], header[0x207]]) >= 0x020c && header[0x236] & 1 != 0 { + return Ok(()); + } + return Err(format!( + "QEMU kernel bzImage must target x86_64: {}", + path.display() + )); + } + Err(format!( + "unsupported QEMU kernel format in {}; expected x86_64 ELF or bzImage", + path.display() + )) +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct VsockPortMap { pub guest_port: u32, @@ -61,6 +138,11 @@ pub struct VmLaunchConfig { } pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { + // Both workload and image-preparation launchers enter here. Reject + // incompatible external kernels before touching disks or starting the VMM. + if let Some(kernel_image) = &config.kernel_image { + config.backend.validate_kernel_image(kernel_image)?; + } match config.backend { VmBackend::Qemu => run_qemu_vm(config), VmBackend::Libkrun => run_libkrun_vm(config), @@ -71,13 +153,12 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { if !config.mounts.is_empty() { return Err("virtiofs mounts are not yet supported with the QEMU backend".to_string()); } - let gpu_bdf = config - .gpu_bdf - .as_deref() - .ok_or("gpu_bdf is required for QEMU backend")?; - let vsock_cid = config - .vsock_cid - .ok_or("vsock_cid is required for QEMU backend")?; + if config.vsock_port_map.is_some() { + return Err( + "QEMU uses host AF_VSOCK; Unix-socket vsock port maps require libkrun".to_string(), + ); + } + VmBackend::Qemu.validate_host()?; if !config.root_disk.is_file() { return Err(format!( @@ -97,16 +178,6 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { return Err(format!("image disk not found: {}", image_disk.display())); } - if let Err(err) = procguard::die_with_parent_cleanup(procguard_kill_children) { - return Err(format!("procguard arm failed: {err}")); - } - - #[cfg(target_os = "linux")] - check_kvm_access()?; - - let guest_env = qemu_guest_env_vars(config); - write_guest_env_file(&config.overlay_disk, &guest_env)?; - let vmlinux = if let Some(kernel_image) = &config.kernel_image { kernel_image.clone() } else { @@ -115,43 +186,18 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { if !vmlinux.is_file() { return Err(format!("VM kernel not found: {}", vmlinux.display())); } + validate_qemu_kernel_image(&vmlinux)?; + let mut qemu_cmd = qemu_command(config, &vmlinux)?; - let kernel_cmdline = build_kernel_cmdline(config); + if let Err(err) = procguard::die_with_parent_cleanup(procguard_kill_children) { + return Err(format!("procguard arm failed: {err}")); + } - let mut qemu_cmd = StdCommand::new("qemu-system-x86_64"); - qemu_cmd - .arg("-machine") - .arg("q35,accel=kvm") - .arg("-cpu") - .arg("host") - .arg("-smp") - .arg(config.vcpus.to_string()) - .arg("-m") - .arg(format!("{}M", config.mem_mib)) - .arg("-nographic") - .arg("-no-reboot") - .args(qemu_network_args()) - .arg("-kernel") - .arg(&vmlinux) - .arg("-append") - .arg(&kernel_cmdline) - .args(qemu_disk_args(config)) - .arg("-device") - .arg("pcie-root-port,id=vsock_root,slot=1") - .arg("-device") - .arg(format!( - "vhost-vsock-pci,guest-cid={vsock_cid},bus=vsock_root" - )) - .arg("-device") - .arg("pcie-root-port,id=gpu_root,slot=2") - .arg("-device") - .arg(format!("vfio-pci,host={gpu_bdf},bus=gpu_root")) - .arg("-serial") - .arg(format!("file:{}", config.console_output.display())); + #[cfg(target_os = "linux")] + check_kvm_access()?; - qemu_cmd.stdin(Stdio::null()); - qemu_cmd.stdout(Stdio::inherit()); - qemu_cmd.stderr(Stdio::inherit()); + let guest_env = qemu_guest_env_vars(config); + write_guest_env_file(&config.overlay_disk, &guest_env)?; #[cfg(target_os = "linux")] { @@ -185,6 +231,55 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { } } +fn qemu_command(config: &VmLaunchConfig, kernel: &Path) -> Result { + let vsock_cid = config + .vsock_cid + .ok_or("vsock_cid is required for QEMU backend")?; + if vsock_cid < 3 { + return Err("QEMU guest vsock CID must be at least 3".to_string()); + } + let mut qemu_cmd = StdCommand::new("qemu-system-x86_64"); + qemu_cmd + .arg("-machine") + .arg("q35,accel=kvm") + .arg("-cpu") + .arg("host") + .arg("-smp") + .arg(config.vcpus.to_string()) + .arg("-m") + .arg(format!("{}M", config.mem_mib)) + .arg("-nographic") + .arg("-no-reboot") + .args(qemu_network_args()) + .arg("-kernel") + .arg(kernel) + .arg("-append") + .arg(build_kernel_cmdline(config)) + .args(qemu_disk_args(config)) + .arg("-device") + .arg("pcie-root-port,id=vsock_root,slot=1") + .arg("-device") + .arg(format!( + "vhost-vsock-pci,guest-cid={vsock_cid},bus=vsock_root" + )) + .arg("-serial") + .arg(format!("file:{}", config.console_output.display())); + + if let Some(gpu_bdf) = &config.gpu_bdf { + qemu_cmd + .arg("-device") + .arg("pcie-root-port,id=gpu_root,slot=2") + .arg("-device") + .arg(format!("vfio-pci,host={gpu_bdf},bus=gpu_root")); + } + + qemu_cmd.stdin(Stdio::null()); + qemu_cmd.stdout(Stdio::inherit()); + qemu_cmd.stderr(Stdio::inherit()); + + Ok(qemu_cmd) +} + fn qemu_network_args() -> [&'static str; 2] { ["-nic", "none"] } @@ -825,6 +920,129 @@ mod tests { } } + #[test] + fn qemu_command_keeps_vsock_and_only_adds_vfio_for_an_assigned_gpu() { + let mut config = qemu_config(); + for gpu in [None, Some("0000:01:00.0".to_string())] { + config.gpu_bdf = gpu.clone(); + let command = qemu_command(&config, Path::new("/vmlinux")).expect("QEMU command"); + let args: Vec<_> = command + .get_args() + .map(|arg| arg.to_string_lossy()) + .collect(); + assert!( + args.iter() + .any(|arg| arg == "vhost-vsock-pci,guest-cid=4,bus=vsock_root") + ); + assert!(args.windows(2).any(|pair| pair == ["-nic", "none"])); + assert_eq!( + args.iter().any(|arg| arg.starts_with("vfio-pci,")), + gpu.is_some() + ); + assert_eq!( + args.iter().any(|arg| arg.contains("gpu_root,slot=2")), + gpu.is_some() + ); + assert_eq!( + qemu_guest_env_vars(&config) + .iter() + .any(|env| env == "GPU_ENABLED=true"), + gpu.is_some() + ); + } + for cid in [None, Some(2)] { + config.vsock_cid = cid; + assert!(qemu_command(&config, Path::new("/vmlinux")).is_err()); + } + } + + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + #[test] + fn qemu_rejects_unsupported_hosts_before_disk_setup() { + let mut config = qemu_config(); + config.gpu_bdf = None; + assert_eq!( + run_vm(&config).unwrap_err(), + "QEMU requires a Linux x86_64 host with KVM" + ); + } + + #[test] + fn kernel_preflight_checks_format_and_guest_architecture() { + let file = tempfile::NamedTempFile::new().unwrap(); + let mut elf = vec![0u8; 64]; + elf[..6].copy_from_slice(b"\x7fELF\x02\x01"); + elf[18] = 62; // EM_X86_64. + std::fs::write(file.path(), &elf).unwrap(); + VmBackend::Qemu + .validate_kernel_image(file.path()) + .expect("x86_64 ELF header"); + elf[18] = 183; // EM_AARCH64. + std::fs::write(file.path(), &elf).unwrap(); + assert!( + VmBackend::Qemu + .validate_kernel_image(file.path()) + .unwrap_err() + .contains("x86_64 ELF") + ); + + let mut bzimage = vec![0u8; 0x238]; + bzimage[0x1fe..0x200].copy_from_slice(&[0x55, 0xaa]); + bzimage[0x202..0x206].copy_from_slice(b"HdrS"); + bzimage[0x206..0x208].copy_from_slice(&0x020cu16.to_le_bytes()); + bzimage[0x236] = 1; + std::fs::write(file.path(), &bzimage).unwrap(); + VmBackend::Qemu + .validate_kernel_image(file.path()) + .expect("x86_64 bzImage header"); + bzimage[0x236] = 0; + std::fs::write(file.path(), &bzimage).unwrap(); + assert!( + VmBackend::Qemu + .validate_kernel_image(file.path()) + .unwrap_err() + .contains("bzImage must target x86_64") + ); + for invalid in [b"".as_slice(), b"\x7fELF", b"not a kernel"] { + std::fs::write(file.path(), invalid).unwrap(); + assert!(VmBackend::Qemu.validate_kernel_image(file.path()).is_err()); + } + } + + #[test] + fn workload_and_preparation_entry_reject_incompatible_kernels_before_disk_setup() { + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(file.path(), b"not a kernel").unwrap(); + for mode in ["sandbox", "image-prep"] { + let mut config = qemu_config(); + config.kernel_image = Some(file.path().to_path_buf()); + config.gpu_bdf = None; + config.env = vec![format!("OPENSHELL_VM_INIT_MODE={mode}")]; + assert!( + run_vm(&config) + .unwrap_err() + .contains("unsupported QEMU kernel format") + ); + config.backend = VmBackend::Libkrun; + assert!( + run_vm(&config) + .unwrap_err() + .contains("libkrun does not support external kernel images") + ); + } + let mut config = qemu_config(); + config.vsock_port_map = Some(VsockPortMap { + guest_port: 1234, + host_socket: "/tmp/unused.sock".into(), + host_initiated: false, + }); + assert!( + run_vm(&config) + .unwrap_err() + .contains("Unix-socket vsock port maps require libkrun") + ); + } + #[test] fn qemu_guest_env_vars_omit_network_metadata() { let env = qemu_guest_env_vars(&qemu_config()); diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 70f6891ce1..30ae5b32d5 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -214,6 +214,12 @@ Podman also supports `image` mounts, which mount a container image read-only at The MicroVM driver runs each sandbox in its own lightweight VM. It requires host virtualization: Apple Hypervisor on macOS or KVM on Linux. +CPU-only VMs use libkrun by default. On Linux x86_64 hosts with KVM, +operator-installed lifecycle extensions can select QEMU when they need an +external x86_64 kernel, without requesting a GPU. +QEMU VM launches don't support virtiofs mounts. Kernel selection remains under +operator control; the guest must still pass authenticated runtime admission. + The VM driver is opt-in. Enable it in the gateway TOML file: ```toml From 6de28d6d4e74917e6d67f241dca778c64aeff530 Mon Sep 17 00:00:00 2001 From: Shiju Date: Wed, 18 Nov 2026 19:14:53 +0530 Subject: [PATCH 3/3] fix(vm): preserve finalized launch plans through lifecycle hooks Signed-off-by: Shiju --- crates/openshell-driver-vm/README.md | 28 +- crates/openshell-driver-vm/src/driver.rs | 335 +++++++------------- crates/openshell-driver-vm/src/lifecycle.rs | 102 ++++-- crates/openshell-driver-vm/src/runtime.rs | 106 ++++--- 4 files changed, 266 insertions(+), 305 deletions(-) diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index e9394df901..e35368641b 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -27,11 +27,10 @@ and `require_backend_feature`. An explicit backend stays selected; incompatible requirements fail before the workload launcher starts. Without an explicit requirement, the driver chooses a backend that supports the requested features. -These are VM-driver-local backend capabilities. The driver resolves the final -plan after operator-installed extensions contribute their requirements, then -checks it against the selected backend. The gateway's `GetCapabilities` -snapshot describes the compute-driver protocol and supported resource request -forms; it does not describe this per-launch VM plan. +The VM driver resolves the plan after its lifecycle extensions contribute +requirements, then checks the selected backend. The compute driver's +`GetCapabilities` RPC supplies the gateway with protocol and resource request +support at startup. | Feature | libkrun | QEMU | |---|---|---| @@ -40,20 +39,21 @@ forms; it does not describe this per-launch VM plan. | PCI passthrough | Unsupported | Requires an assigned GPU device | | Host directory mounts | Supported through virtiofs | Unsupported | -For example, an operator-installed extension can set `plan.kernel_image` and -require `BackendFeature::ExternalKernelImage` for a CPU-only sandbox. The +For example, a VM driver with a registered lifecycle extension can set +`plan.kernel_image` for a CPU-only sandbox. The resolver selects QEMU with a vsock CID and ordinary CPU sizing, without allocating a GPU. Requiring libkrun for that same plan fails with an external-kernel diagnostic. Kernel paths remain extension-owned; sandbox requests cannot supply them. -The current QEMU launcher requires a Linux x86_64 host with KVM. Other host -targets fail preflight before the workload launcher starts. External kernels are checked for a -64-bit little-endian x86_64 ELF header or an x86_64 Linux bzImage header. The -VMM validates the complete image. Kernel security and boot-driver requirements -still apply, and authenticated runtime admission confirms enforcement before -workload code runs. Both workload and image-preparation launchers use the same -kernel preflight entry point; preparation retains its libkrun bootstrap backend. +The QEMU launcher requires a Linux x86_64 host with KVM and vhost-vsock. +Supply a kernel through a lifecycle extension or place `vmlinux` in the +directory configured by `OPENSHELL_VM_RUNTIME_DIR`. The driver checks the +selected kernel's x86_64 ELF or Linux bzImage header before starting the +workload launcher. QEMU validates the complete image and requires a PVH entry +note for an uncompressed Linux ELF kernel. Kernel security and boot-driver +requirements still apply. Authenticated runtime admission confirms +enforcement before workload code runs. Image preparation uses libkrun. ## How it fits together diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 2938a22245..7cd7914a63 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1936,16 +1936,7 @@ impl VmDriver { None }; - let needs_qemu = is_gpu; - - let mut plan = - match self.build_vm_launch_plan(&sandbox.id, needs_qemu, is_gpu, gpu_bdf.clone()) { - Ok(plan) => plan, - Err(err) => { - self.release_gpu(&sandbox.id); - return Err(err); - } - }; + let mut plan = self.build_vm_launch_plan(gpu_bdf.clone()); if let Err(err) = self .lifecycle_extensions @@ -1971,50 +1962,38 @@ impl VmDriver { }); } - // Resolve and validate the backend from the requirements that - // `configure_launch` extensions contributed. After this point the - // plan's backend, sizing, and host allocations are final; the - // `before_launch` hook below may still mutate - // `plan.env` and `plan.guest_init_dropins` and may abort the launch, - // but it MUST NOT change `plan.backend`, `plan.required_backends`, - // or `plan.required_backend_features` -- those are enforced as a - // documented trait contract, not a runtime check. - if let Err(err) = - self.resolve_launch_plan_backend(&sandbox.id, is_gpu, gpu_bdf.clone(), &mut plan) - { - self.lifecycle_extensions - .after_launch_failed( - &sandbox, - &state_dir, - LaunchAbortReason::BeforeLaunchHookFailed, - ) - .await; - self.release_gpu(&sandbox.id); - return Err(err); - } - - if let Err(err) = Self::validate_launch_plan_backend(is_gpu, &plan).and_then(|()| { - plan.backend - .validate_host() - .map_err(Status::failed_precondition)?; - // Extensions can select QEMU after request validation, so check - // the selected backend's mount support again before launch. - validate_vm_driver_mounts( - &driver_config.mounts, - &self.config, - plan.backend == VmBackend::Qemu, - ) - }) { - self.lifecycle_extensions - .after_launch_failed( - &sandbox, - &state_dir, - LaunchAbortReason::BeforeLaunchHookFailed, - ) - .await; - self.release_gpu(&sandbox.id); - return Err(err); - } + // Resolve extension requirements before before_launch prepares the host. + let kernel_image = self + .resolve_launch_plan_backend(is_gpu, &mut plan) + .and_then(|()| Self::validate_launch_plan_backend(gpu_bdf.as_deref(), &plan)) + .and_then(|()| { + plan.backend + .validate_host() + .map_err(Status::failed_precondition)?; + // An extension can select QEMU after request validation. + validate_vm_driver_mounts( + &driver_config.mounts, + &self.config, + plan.backend == VmBackend::Qemu, + )?; + plan.backend + .resolve_kernel_image(plan.kernel_image.as_deref()) + .map_err(Status::failed_precondition) + }); + let kernel_image = match kernel_image { + Ok(path) => path, + Err(err) => { + self.lifecycle_extensions + .after_launch_failed( + &sandbox, + &state_dir, + LaunchAbortReason::BeforeLaunchHookFailed, + ) + .await; + self.release_gpu(&sandbox.id); + return Err(err); + } + }; if let Err(err) = self .lifecycle_extensions @@ -2156,7 +2135,7 @@ impl VmDriver { command.arg("--vm-console-output").arg(&console_output); command.arg("--vm-vcpus").arg(plan.vcpus.to_string()); command.arg("--vm-mem-mib").arg(plan.mem_mib.to_string()); - if let Some(kernel_image) = &plan.kernel_image { + if let Some(kernel_image) = &kernel_image { command.arg("--vm-kernel-image").arg(kernel_image); } @@ -2993,9 +2972,7 @@ impl VmDriver { #[allow(clippy::result_large_err)] fn resolve_launch_plan_backend( &self, - sandbox_id: &str, is_gpu: bool, - gpu_bdf: Option, plan: &mut LaunchPlan, ) -> Result<(), Status> { if plan.kernel_image.is_some() { @@ -3029,71 +3006,29 @@ impl VmDriver { } if plan.backend == VmBackend::Qemu { - self.configure_qemu_launch_plan(sandbox_id, is_gpu, gpu_bdf, plan)?; + if is_gpu { + plan.vcpus = self.config.gpu_vcpus; + plan.mem_mib = self.config.gpu_mem_mib; + } + plan.vsock_cid.get_or_insert_with(allocate_vsock_cid); } Ok(()) } - // Keep the fallible shape used by launch-plan resolution: driver-local - // backends may add allocation failures here without changing callers. - #[allow(clippy::result_large_err, clippy::unnecessary_wraps)] - fn configure_qemu_launch_plan( - &self, - _sandbox_id: &str, - is_gpu: bool, - gpu_bdf: Option, - plan: &mut LaunchPlan, - ) -> Result<(), Status> { - plan.backend = VmBackend::Qemu; - if is_gpu { - plan.vcpus = self.config.gpu_vcpus; - plan.mem_mib = self.config.gpu_mem_mib; - } - if plan.gpu_bdf.is_none() { - plan.gpu_bdf = gpu_bdf; - } - if plan.vsock_cid.is_some() { - return Ok(()); - } - plan.vsock_cid = Some(allocate_vsock_cid()); - Ok(()) - } - #[allow(clippy::result_large_err)] - fn validate_launch_plan_backend(is_gpu: bool, plan: &LaunchPlan) -> Result<(), Status> { - if plan.backend != VmBackend::Qemu && is_gpu { - return Err(Status::failed_precondition( - "GPU sandbox launch requires the QEMU backend", - )); - } - if plan.gpu_bdf.is_some() && !is_gpu { - return Err(Status::failed_precondition( - "GPU passthrough requires a GPU sandbox request", - )); - } - if is_gpu && plan.gpu_bdf.is_none() { - return Err(Status::failed_precondition( - "GPU sandbox launch is missing its assigned GPU device", - )); - } - if plan.required_backends.contains(&VmBackend::Libkrun) - && plan.required_backends.contains(&VmBackend::Qemu) - { - return Err(Status::failed_precondition( - "VM lifecycle extensions requested conflicting VM backends", - )); - } - if plan.required_backends.contains(&VmBackend::Libkrun) - && plan.backend != VmBackend::Libkrun - { + fn validate_launch_plan_backend( + assigned_gpu: Option<&str>, + plan: &LaunchPlan, + ) -> Result<(), Status> { + if plan.gpu_bdf.as_deref() != assigned_gpu { return Err(Status::failed_precondition( - "VM lifecycle extension requires the libkrun backend", + "VM lifecycle extensions must preserve the driver's GPU assignment", )); } - if plan.required_backends.contains(&VmBackend::Qemu) && plan.backend != VmBackend::Qemu { + if assigned_gpu.is_some() && plan.backend != VmBackend::Qemu { return Err(Status::failed_precondition( - "VM lifecycle extension requires the QEMU backend", + "GPU sandbox launch requires the QEMU backend", )); } if let Some(feature) = plan @@ -3110,55 +3045,34 @@ impl VmDriver { if plan .required_backend_features .contains(&BackendFeature::PciPassthrough) - && plan.gpu_bdf.is_none() + && assigned_gpu.is_none() { return Err(Status::failed_precondition( "PCI passthrough requires an assigned GPU device; arbitrary PCI devices are not supported", )); } - if let Some(kernel_image) = &plan.kernel_image { - plan.backend - .validate_kernel_image(kernel_image) - .map_err(Status::failed_precondition)?; - } Ok(()) } - // Keep the fallible shape used by provisioning and lifecycle tests even - // though NIC/subnet allocation no longer introduces a failure today. - #[allow(clippy::result_large_err, clippy::unnecessary_wraps)] - fn build_vm_launch_plan( - &self, - _sandbox_id: &str, - needs_qemu: bool, - is_gpu: bool, - gpu_bdf: Option, - ) -> Result { - if !needs_qemu { - return Ok(LaunchPlan { - backend: VmBackend::Libkrun, - vcpus: self.config.vcpus, - mem_mib: self.config.mem_mib, - required_backends: Vec::new(), - required_backend_features: Vec::new(), - kernel_profile: None, - kernel_image: None, - gpu_bdf: None, - vsock_cid: None, - guest_init_dropins: Vec::new(), - env: Vec::new(), - }); - } - - let vsock_cid = allocate_vsock_cid(); - let (vcpus, mem_mib) = if is_gpu { - (self.config.gpu_vcpus, self.config.gpu_mem_mib) + fn build_vm_launch_plan(&self, gpu_bdf: Option) -> LaunchPlan { + let (backend, vcpus, mem_mib, vsock_cid) = if gpu_bdf.is_some() { + ( + VmBackend::Qemu, + self.config.gpu_vcpus, + self.config.gpu_mem_mib, + Some(allocate_vsock_cid()), + ) } else { - (self.config.vcpus, self.config.mem_mib) + ( + VmBackend::Libkrun, + self.config.vcpus, + self.config.mem_mib, + None, + ) }; - Ok(LaunchPlan { - backend: VmBackend::Qemu, + LaunchPlan { + backend, vcpus, mem_mib, required_backends: Vec::new(), @@ -3166,10 +3080,10 @@ impl VmDriver { kernel_profile: None, kernel_image: None, gpu_bdf, - vsock_cid: Some(vsock_cid), + vsock_cid, guest_init_dropins: Vec::new(), env: Vec::new(), - }) + } } async fn ensure_provisioning_active(&self, sandbox_id: &str) -> Result<(), Status> { @@ -9457,9 +9371,7 @@ mod tests { id: "sb-post-overlay".to_string(), ..Default::default() }; - let mut plan = driver - .build_vm_launch_plan(&sandbox.id, false, false, None) - .unwrap(); + let mut plan = driver.build_vm_launch_plan(None); let provisioning = tracing::info_span!("vm.provision"); async { @@ -13644,9 +13556,11 @@ while :; do sleep 0.05; done"#, fn empty_registry_keeps_non_gpu_sandbox_on_libkrun() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let plan = driver - .build_vm_launch_plan("sandbox-x", false, false, None) - .expect("plan should build"); + let mut plan = driver.build_vm_launch_plan(None); + driver + .resolve_launch_plan_backend(false, &mut plan) + .unwrap(); + VmDriver::validate_launch_plan_backend(None, &plan).unwrap(); assert_eq!(plan.backend, VmBackend::Libkrun); assert_eq!(plan.vcpus, 2); @@ -13666,81 +13580,79 @@ while :; do sleep 0.05; done"#, fn gpu_sandbox_uses_qemu_backend_and_gpu_sizing() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let plan = driver - .build_vm_launch_plan("sandbox-gpu", true, true, Some("0000:01:00.0".to_string())) - .expect("gpu plan should build"); + let assigned = "0000:01:00.0"; + let mut plan = driver.build_vm_launch_plan(Some(assigned.to_string())); + driver.resolve_launch_plan_backend(true, &mut plan).unwrap(); + VmDriver::validate_launch_plan_backend(Some(assigned), &plan).unwrap(); assert_eq!(plan.backend, VmBackend::Qemu); assert_eq!(plan.vcpus, 8); assert_eq!(plan.mem_mib, 16384); - assert_eq!(plan.gpu_bdf.as_deref(), Some("0000:01:00.0")); + assert_eq!(plan.gpu_bdf.as_deref(), Some(assigned)); assert!(plan.vsock_cid.is_some()); + + for replacement in [None, Some("0000:02:00.0".to_string())] { + plan.gpu_bdf = replacement; + driver.resolve_launch_plan_backend(true, &mut plan).unwrap(); + let error = VmDriver::validate_launch_plan_backend(Some(assigned), &plan) + .expect_err("the launch plan must keep the device reserved by the driver"); + assert!(error.message().contains("GPU assignment")); + } } #[test] - fn launch_plan_rejects_external_kernel_on_unsupported_backend() { - let mut plan = LaunchPlan { - backend: VmBackend::Libkrun, - vcpus: 2, - mem_mib: 2048, - required_backends: Vec::new(), - required_backend_features: Vec::new(), - kernel_profile: None, - kernel_image: Some(PathBuf::from("/tmp/openshell-test-kernel")), - gpu_bdf: None, - vsock_cid: None, - guest_init_dropins: Vec::new(), - env: Vec::new(), - }; - - let err = VmDriver::validate_launch_plan_backend(false, &plan) - .expect_err("external kernels require a compatible backend"); - assert_eq!(err.code(), Code::FailedPrecondition); - assert!(err.message().contains("external kernel images")); - - let base = unique_temp_dir(); - std::fs::create_dir_all(&base).unwrap(); - let kernel = base.join("vmlinux"); + fn external_kernel_selects_cpu_only_qemu_unless_libkrun_is_required() { + let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + let mut plan = driver.build_vm_launch_plan(None); + let kernel = tempfile::NamedTempFile::new().unwrap(); let mut header = [0u8; 64]; header[..6].copy_from_slice(b"\x7fELF\x02\x01"); header[18] = 62; // EM_X86_64; a header fixture, not a bootable guest. - std::fs::write(&kernel, header).unwrap(); - plan.kernel_image = Some(kernel); - let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + std::fs::write(kernel.path(), header).unwrap(); + plan.kernel_image = Some(kernel.path().to_path_buf()); plan.require_backend(VmBackend::Libkrun); driver - .resolve_launch_plan_backend("kernel-check", false, None, &mut plan) + .resolve_launch_plan_backend(false, &mut plan) .unwrap(); assert_eq!(plan.backend, VmBackend::Libkrun); - assert!(VmDriver::validate_launch_plan_backend(false, &plan).is_err()); + let error = VmDriver::validate_launch_plan_backend(None, &plan) + .expect_err("an explicit libkrun requirement rules out external kernels"); + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("external-kernel-image")); plan.required_backends.clear(); driver - .resolve_launch_plan_backend("kernel-check", false, None, &mut plan) + .resolve_launch_plan_backend(false, &mut plan) .unwrap(); assert_eq!(plan.backend, VmBackend::Qemu); + assert_eq!(plan.vcpus, driver.config.vcpus); + assert_eq!(plan.mem_mib, driver.config.mem_mib); assert!(plan.gpu_bdf.is_none()); assert!(plan.vsock_cid.is_some()); - VmDriver::validate_launch_plan_backend(false, &plan) + VmDriver::validate_launch_plan_backend(None, &plan) .expect("CPU-only external kernel is accepted without PCI passthrough"); - let _ = std::fs::remove_dir_all(base); + assert_eq!( + plan.backend + .resolve_kernel_image(plan.kernel_image.as_deref()) + .unwrap() + .as_deref(), + Some(kernel.path()) + ); } #[test] fn backend_feature_requirements_select_qemu_launch_plan() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let mut plan = driver - .build_vm_launch_plan("sandbox-vfio", false, false, None) - .expect("base plan should build"); + let mut plan = driver.build_vm_launch_plan(None); plan.require_backend_feature(BackendFeature::PciPassthrough); driver - .resolve_launch_plan_backend("sandbox-vfio", false, None, &mut plan) + .resolve_launch_plan_backend(false, &mut plan) .expect("backend feature should resolve"); assert_eq!(plan.backend, VmBackend::Qemu); assert!(plan.vsock_cid.is_some()); - let error = VmDriver::validate_launch_plan_backend(false, &plan) + let error = VmDriver::validate_launch_plan_backend(None, &plan) .expect_err("a PCI feature needs an assigned device"); assert!(error.message().contains("assigned GPU device")); } @@ -13748,30 +13660,27 @@ while :; do sleep 0.05; done"#, #[test] fn explicit_backend_requirement_selects_qemu_launch_plan() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let mut plan = driver - .build_vm_launch_plan("sandbox-qemu", false, false, None) - .expect("base plan should build"); + let mut plan = driver.build_vm_launch_plan(None); plan.require_backend(VmBackend::Qemu); driver - .resolve_launch_plan_backend("sandbox-qemu", false, None, &mut plan) + .resolve_launch_plan_backend(false, &mut plan) .expect("backend requirement should resolve"); assert_eq!(plan.backend, VmBackend::Qemu); assert!(plan.vsock_cid.is_some()); - VmDriver::validate_launch_plan_backend(false, &plan).expect("CPU-only QEMU is supported"); + VmDriver::validate_launch_plan_backend(None, &plan) + .expect("CPU-only QEMU can satisfy the backend requirements"); } #[test] fn conflicting_backend_requirements_fail_before_allocating_a_cid() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let mut plan = driver - .build_vm_launch_plan("conflict", false, false, None) - .unwrap(); + let mut plan = driver.build_vm_launch_plan(None); plan.require_backend(VmBackend::Libkrun); plan.require_backend(VmBackend::Qemu); let error = driver - .resolve_launch_plan_backend("conflict", false, None, &mut plan) + .resolve_launch_plan_backend(false, &mut plan) .expect_err("conflicting runtimes must fail"); assert_eq!(error.code(), Code::FailedPrecondition); assert!(error.message().contains("conflicting VM backends")); @@ -13781,13 +13690,11 @@ while :; do sleep 0.05; done"#, #[test] fn guest_init_dropin_feature_does_not_force_qemu() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let mut plan = driver - .build_vm_launch_plan("sandbox-init", false, false, None) - .expect("base plan should build"); + let mut plan = driver.build_vm_launch_plan(None); plan.require_backend_feature(BackendFeature::GuestInitDropins); driver - .resolve_launch_plan_backend("sandbox-init", false, None, &mut plan) + .resolve_launch_plan_backend(false, &mut plan) .expect("guest init feature should resolve"); assert_eq!(plan.backend, VmBackend::Libkrun); @@ -14087,11 +13994,9 @@ while :; do sleep 0.05; done"#, #[test] fn qemu_launch_plan_uses_vsock_only_with_host_proxy() { let driver = test_driver_with_proxy("http://127.0.0.1:8080"); - let mut plan = driver - .build_vm_launch_plan("sandbox-proxy-vsock", true, true, None) - .expect("gpu plan should build"); + let mut plan = driver.build_vm_launch_plan(Some("0000:01:00.0".to_string())); driver - .resolve_launch_plan_backend("sandbox-proxy-vsock", true, None, &mut plan) + .resolve_launch_plan_backend(true, &mut plan) .expect("host control can reach a host-loopback proxy"); assert!(plan.vsock_cid.is_some()); } diff --git a/crates/openshell-driver-vm/src/lifecycle.rs b/crates/openshell-driver-vm/src/lifecycle.rs index 6ce929f6a4..b99891ebb7 100644 --- a/crates/openshell-driver-vm/src/lifecycle.rs +++ b/crates/openshell-driver-vm/src/lifecycle.rs @@ -278,7 +278,7 @@ pub enum ExtensionActivation { /// and allocates backend-specific host resources such as a vsock CID. /// 3. [`before_launch`](Self::before_launch) — perform host-side /// side effects with the resolved plan in hand, optionally append -/// additional guest env via [`LaunchPlan::env`]. +/// guest environment entries or init drop-ins. /// 4. The driver spawns the VM launcher process. /// /// On launch failure or sandbox deletion, the driver invokes @@ -329,22 +329,13 @@ pub trait LifecycleExtension: std::fmt::Debug + Send + Sync { /// Perform host-side preparation with the resolved launch plan. /// - /// At this point [`LaunchPlan::backend`], - /// [`LaunchPlan::required_backends`], and - /// [`LaunchPlan::required_backend_features`] are finalized and any - /// backend-specific host resources have been - /// allocated. This hook is the right place to bind PCI devices, set - /// up filesystem state, or otherwise prepare the host. + /// The driver has finalized the backend, requirements, kernel, sizing, + /// GPU assignment, and vsock CID. Use this hook to bind a reserved PCI + /// device, set up filesystem state, or otherwise prepare the host. /// - /// Implementations MAY append entries to [`LaunchPlan::env`] to - /// inject additional guest environment variables, and MAY return an - /// error to abort the launch. Implementations MUST NOT change - /// [`LaunchPlan::backend`], [`LaunchPlan::required_backends`], or - /// [`LaunchPlan::required_backend_features`]; the driver has already - /// resolved and validated them and does not repeat those steps after - /// this hook. - /// The driver-owned [`LaunchPlan::gpu_bdf`] must remain unchanged; - /// the registry rejects hooks that clear or replace that assignment. + /// Implementations can append entries to [`LaunchPlan::env`] and + /// [`LaunchPlan::guest_init_dropins`] or return an error to abort the + /// launch. The registry rejects changes to finalized launch fields. /// /// If this hook performs allocations that must be released on failure /// or delete, implement @@ -599,6 +590,14 @@ impl LifecycleExtensionRegistry { ) -> LifecycleResult<()> { let span_status = openshell_otel::ErrorStatusGuard::current(); let assigned_gpu = plan.gpu_bdf.clone(); + let backend = plan.backend.clone(); + let required_backends = plan.required_backends.clone(); + let required_backend_features = plan.required_backend_features.clone(); + let kernel_profile = plan.kernel_profile.clone(); + let kernel_image = plan.kernel_image.clone(); + let vsock_cid = plan.vsock_cid; + let vcpus = plan.vcpus; + let mem_mib = plan.mem_mib; for ext in self.active_for(sandbox) { ext.before_launch(sandbox, state_dir, plan).await?; if plan.gpu_bdf != assigned_gpu { @@ -607,6 +606,20 @@ impl LifecycleExtensionRegistry { ext.name() ))); } + if plan.backend != backend + || plan.required_backends != required_backends + || plan.required_backend_features != required_backend_features + || plan.kernel_profile != kernel_profile + || plan.kernel_image != kernel_image + || plan.vsock_cid != vsock_cid + || plan.vcpus != vcpus + || plan.mem_mib != mem_mib + { + return Err(LifecycleError::new(format!( + "extension '{}' changed finalized launch fields in before_launch", + ext.name() + ))); + } } span_status.finish(Ok(())) } @@ -907,6 +920,10 @@ mod tests { ))); } plan.env.push(format!("RECORDING_{}=1", self.name)); + plan.guest_init_dropins.push(GuestInitDropin::new( + format!("90-{}.sh", self.name), + b"#!/bin/sh\n".to_vec(), + )); Ok(()) } @@ -979,13 +996,15 @@ mod tests { extension.clone() } + type PlanMutation = fn(&mut LaunchPlan); + #[derive(Debug)] - struct GpuMutatingExtension(Option); + struct PlanMutatingExtension(PlanMutation); #[tonic::async_trait] - impl LifecycleExtension for GpuMutatingExtension { + impl LifecycleExtension for PlanMutatingExtension { fn name(&self) -> &'static str { - "gpu-mutator" + "plan-mutator" } async fn before_launch( @@ -994,23 +1013,47 @@ mod tests { _state_dir: &Path, plan: &mut LaunchPlan, ) -> LifecycleResult<()> { - plan.gpu_bdf.clone_from(&self.0); + (self.0)(plan); Ok(()) } } #[tokio::test] - async fn before_launch_rejects_clearing_or_replacing_the_assigned_gpu() { - for replacement in [None, Some("0000:02:00.0".to_string())] { + async fn before_launch_rejects_changes_to_finalized_fields() { + let mutations: [(&str, PlanMutation); 10] = [ + ("assigned GPU device", |plan| plan.gpu_bdf = None), + ("assigned GPU device", |plan| { + plan.gpu_bdf = Some("0000:02:00.0".to_string()); + }), + ("finalized launch fields", |plan| { + plan.backend = VmBackend::Libkrun; + }), + ("finalized launch fields", |plan| { + plan.require_backend(VmBackend::Libkrun); + }), + ("finalized launch fields", |plan| { + plan.require_backend_feature(BackendFeature::PciPassthrough); + }), + ("finalized launch fields", |plan| { + plan.kernel_profile = Some("other".into()); + }), + ("finalized launch fields", |plan| { + plan.kernel_image = Some("other".into()); + }), + ("finalized launch fields", |plan| plan.vsock_cid = Some(8)), + ("finalized launch fields", |plan| plan.vcpus = 4), + ("finalized launch fields", |plan| plan.mem_mib = 4096), + ]; + for (message, mutate) in mutations { let registry = - LifecycleExtensionRegistry::with(vec![Arc::new(GpuMutatingExtension(replacement))]); + LifecycleExtensionRegistry::with(vec![Arc::new(PlanMutatingExtension(mutate))]); let mut plan = sample_plan(VmBackend::Qemu); plan.gpu_bdf = Some("0000:01:00.0".to_string()); let error = registry .before_launch(&sample_sandbox(), Path::new("/tmp/state"), &mut plan) .await - .expect_err("a requested GPU must retain its assignment after hooks"); - assert!(error.message().contains("changed the assigned GPU device")); + .expect_err("finalized launch fields must survive before_launch"); + assert!(error.message().contains(message)); } } @@ -1132,7 +1175,7 @@ mod tests { } #[tokio::test] - async fn before_launch_runs_each_extension_in_order_and_collects_env() { + async fn before_launch_runs_each_extension_in_order_and_collects_guest_inputs() { let ext_a = RecordingExtension::new("a"); let ext_b = RecordingExtension::new("b"); let registry = @@ -1146,6 +1189,13 @@ mod tests { .expect("before_launch succeeds"); assert_eq!(plan.env, vec!["RECORDING_a=1", "RECORDING_b=1"]); + assert_eq!( + plan.guest_init_dropins + .iter() + .map(|dropin| dropin.name.as_str()) + .collect::>(), + ["90-a.sh", "90-b.sh"] + ); assert_eq!(ext_a.calls(), vec!["a:before_launch"]); assert_eq!(ext_b.calls(), vec!["b:before_launch"]); } diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index ad98899c8e..d74d9aee97 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -35,19 +35,33 @@ impl VmBackend { } } - pub(crate) fn validate_kernel_image(&self, path: &Path) -> Result<(), String> { - match self { - Self::Libkrun => Err(format!( + pub(crate) fn resolve_kernel_image( + &self, + path: Option<&Path>, + ) -> Result, String> { + match (self, path) { + (Self::Libkrun, None) => Ok(None), + (Self::Libkrun, Some(path)) => Err(format!( "libkrun does not support external kernel images: {}", path.display() )), - Self::Qemu => validate_qemu_kernel_image(path), + (Self::Qemu, path) => resolve_qemu_kernel_image(path).map(Some), } } } -/// Preflight the formats accepted by the `x86_64` QEMU launcher. The VMM still -/// validates the complete image; runtime admission confirms enforcement. +fn resolve_qemu_kernel_image(path: Option<&Path>) -> Result { + let path = match path { + Some(path) => path.to_path_buf(), + None => qemu_runtime_dir()?.join("vmlinux"), + }; + validate_qemu_kernel_image(&path)?; + Ok(path) +} + +/// Check the architecture and boot-image header for QEMU direct boot. +/// An uncompressed Linux ELF kernel also needs a PVH entry note, which QEMU +/// checks when it loads the image. fn validate_qemu_kernel_image(path: &Path) -> Result<(), String> { if !path.is_file() { return Err(format!( @@ -138,11 +152,6 @@ pub struct VmLaunchConfig { } pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { - // Both workload and image-preparation launchers enter here. Reject - // incompatible external kernels before touching disks or starting the VMM. - if let Some(kernel_image) = &config.kernel_image { - config.backend.validate_kernel_image(kernel_image)?; - } match config.backend { VmBackend::Qemu => run_qemu_vm(config), VmBackend::Libkrun => run_libkrun_vm(config), @@ -159,6 +168,7 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { ); } VmBackend::Qemu.validate_host()?; + let kernel = resolve_qemu_kernel_image(config.kernel_image.as_deref())?; if !config.root_disk.is_file() { return Err(format!( @@ -178,16 +188,7 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { return Err(format!("image disk not found: {}", image_disk.display())); } - let vmlinux = if let Some(kernel_image) = &config.kernel_image { - kernel_image.clone() - } else { - qemu_runtime_dir()?.join("vmlinux") - }; - if !vmlinux.is_file() { - return Err(format!("VM kernel not found: {}", vmlinux.display())); - } - validate_qemu_kernel_image(&vmlinux)?; - let mut qemu_cmd = qemu_command(config, &vmlinux)?; + let mut qemu_cmd = qemu_command(config, &kernel)?; if let Err(err) = procguard::die_with_parent_cleanup(procguard_kill_children) { return Err(format!("procguard arm failed: {err}")); @@ -388,12 +389,7 @@ fn procguard_kill_children() { } fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { - if let Some(kernel_image) = &config.kernel_image { - return Err(format!( - "selected kernel image is not supported by this VM backend: {}", - kernel_image.display() - )); - } + VmBackend::Libkrun.resolve_kernel_image(config.kernel_image.as_deref())?; if !config.root_disk.is_file() { return Err(format!( "root disk image not found: {}", @@ -535,10 +531,10 @@ pub fn configured_runtime_dir() -> Result { } fn qemu_runtime_dir() -> Result { - configured_runtime_dir().map_err(|_| { - "QEMU backend requires OPENSHELL_VM_RUNTIME_DIR to be set (pointing to a directory \ - containing vmlinux). Set the env var or run `mise run vm:setup`." - .to_string() + configured_runtime_dir().map_err(|error| { + format!( + "QEMU requires an external kernel or a vmlinux file in the directory set by {VM_RUNTIME_DIR_ENV}: {error}" + ) }) } @@ -974,14 +970,17 @@ mod tests { elf[..6].copy_from_slice(b"\x7fELF\x02\x01"); elf[18] = 62; // EM_X86_64. std::fs::write(file.path(), &elf).unwrap(); - VmBackend::Qemu - .validate_kernel_image(file.path()) - .expect("x86_64 ELF header"); + assert_eq!( + VmBackend::Qemu + .resolve_kernel_image(Some(file.path())) + .expect("x86_64 ELF header"), + Some(file.path().to_path_buf()) + ); elf[18] = 183; // EM_AARCH64. std::fs::write(file.path(), &elf).unwrap(); assert!( VmBackend::Qemu - .validate_kernel_image(file.path()) + .resolve_kernel_image(Some(file.path())) .unwrap_err() .contains("x86_64 ELF") ); @@ -993,43 +992,50 @@ mod tests { bzimage[0x236] = 1; std::fs::write(file.path(), &bzimage).unwrap(); VmBackend::Qemu - .validate_kernel_image(file.path()) + .resolve_kernel_image(Some(file.path())) .expect("x86_64 bzImage header"); bzimage[0x236] = 0; std::fs::write(file.path(), &bzimage).unwrap(); assert!( VmBackend::Qemu - .validate_kernel_image(file.path()) + .resolve_kernel_image(Some(file.path())) .unwrap_err() .contains("bzImage must target x86_64") ); for invalid in [b"".as_slice(), b"\x7fELF", b"not a kernel"] { std::fs::write(file.path(), invalid).unwrap(); - assert!(VmBackend::Qemu.validate_kernel_image(file.path()).is_err()); + assert!( + VmBackend::Qemu + .resolve_kernel_image(Some(file.path())) + .is_err() + ); } } #[test] - fn workload_and_preparation_entry_reject_incompatible_kernels_before_disk_setup() { + fn launcher_entry_rejects_incompatible_external_kernels_before_disk_setup() { let file = tempfile::NamedTempFile::new().unwrap(); std::fs::write(file.path(), b"not a kernel").unwrap(); - for mode in ["sandbox", "image-prep"] { - let mut config = qemu_config(); - config.kernel_image = Some(file.path().to_path_buf()); - config.gpu_bdf = None; - config.env = vec![format!("OPENSHELL_VM_INIT_MODE={mode}")]; + let mut config = qemu_config(); + config.kernel_image = Some(file.path().to_path_buf()); + config.gpu_bdf = None; + if cfg!(all(target_os = "linux", target_arch = "x86_64")) { assert!( run_vm(&config) .unwrap_err() .contains("unsupported QEMU kernel format") ); - config.backend = VmBackend::Libkrun; - assert!( - run_vm(&config) - .unwrap_err() - .contains("libkrun does not support external kernel images") - ); } + config.backend = VmBackend::Libkrun; + assert!( + run_vm(&config) + .unwrap_err() + .contains("libkrun does not support external kernel images") + ); + } + + #[test] + fn qemu_launcher_rejects_unix_vsock_port_maps() { let mut config = qemu_config(); config.vsock_port_map = Some(VsockPortMap { guest_port: 1234,