From 870dfe359dcef858a4bf5563f07ec8e817857f1a Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Wed, 11 Nov 2026 00:04:56 +0000 Subject: [PATCH 1/8] bhyve-api: bump version for x2apic support --- crates/bhyve-api/src/lib.rs | 7 ++++++- crates/bhyve-api/sys/src/lib.rs | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/bhyve-api/src/lib.rs b/crates/bhyve-api/src/lib.rs index 00965ec13..079d75eea 100644 --- a/crates/bhyve-api/src/lib.rs +++ b/crates/bhyve-api/src/lib.rs @@ -541,6 +541,11 @@ unsafe fn ioctl( #[repr(u32)] #[derive(Copy, Clone)] pub enum ApiVersion { + /// Add support for guest initiated x2APIC mode changes. + /// `VM_SET_X2APIC_STATE` no longer forcibly switches the guest's x2APIC + /// state and only controls whether the x2APIC bit is exposed via the + /// legacy cpuid emulation. + V20 = 20, /// `VM_GET_CPUS` accepts a set of any size up to a page, and fails with /// `ERANGE` if the set is too small to hold every member V19 = 19, @@ -594,7 +599,7 @@ pub enum ApiVersion { } impl ApiVersion { pub const fn current() -> Self { - Self::V19 + Self::V20 } } diff --git a/crates/bhyve-api/sys/src/lib.rs b/crates/bhyve-api/sys/src/lib.rs index 07952d038..84bbf0fba 100644 --- a/crates/bhyve-api/sys/src/lib.rs +++ b/crates/bhyve-api/sys/src/lib.rs @@ -17,4 +17,4 @@ pub const VM_MAXCPU: usize = 32; /// This is the VMM interface version which bhyve_api expects to operate /// against. All constants and structs defined by the crate are done so in /// terms of that specific version. -pub const VMM_CURRENT_INTERFACE_VERSION: u32 = 19; +pub const VMM_CURRENT_INTERFACE_VERSION: u32 = 20; From ebeca441315b85952d5f9cb765f646aff41aac9c Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Wed, 11 Nov 2026 21:18:00 +0000 Subject: [PATCH 2/8] bhyve-api: bump version for Extended MSI destination ID support --- crates/bhyve-api/src/lib.rs | 4 +++- crates/bhyve-api/sys/src/lib.rs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/bhyve-api/src/lib.rs b/crates/bhyve-api/src/lib.rs index 079d75eea..087e931db 100644 --- a/crates/bhyve-api/src/lib.rs +++ b/crates/bhyve-api/src/lib.rs @@ -541,6 +541,8 @@ unsafe fn ioctl( #[repr(u32)] #[derive(Copy, Clone)] pub enum ApiVersion { + /// Add support for 15-bit Extended MSI Destination ID enlightenment + V21 = 21, /// Add support for guest initiated x2APIC mode changes. /// `VM_SET_X2APIC_STATE` no longer forcibly switches the guest's x2APIC /// state and only controls whether the x2APIC bit is exposed via the @@ -599,7 +601,7 @@ pub enum ApiVersion { } impl ApiVersion { pub const fn current() -> Self { - Self::V20 + Self::V21 } } diff --git a/crates/bhyve-api/sys/src/lib.rs b/crates/bhyve-api/sys/src/lib.rs index 84bbf0fba..2c04f1ce5 100644 --- a/crates/bhyve-api/sys/src/lib.rs +++ b/crates/bhyve-api/sys/src/lib.rs @@ -17,4 +17,4 @@ pub const VM_MAXCPU: usize = 32; /// This is the VMM interface version which bhyve_api expects to operate /// against. All constants and structs defined by the crate are done so in /// terms of that specific version. -pub const VMM_CURRENT_INTERFACE_VERSION: u32 = 20; +pub const VMM_CURRENT_INTERFACE_VERSION: u32 = 21; From d863772c146c0c78b2445681f807a4bc1ee40033 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Wed, 11 Nov 2026 21:22:36 +0000 Subject: [PATCH 3/8] Update bhyve enlightenment stack to allow new EXT_MSI_DEST_ID feature --- bin/propolis-server/src/lib/vm/ensure.rs | 2 +- bin/propolis-standalone/src/config.rs | 14 ++- bin/propolis-standalone/src/main.rs | 6 +- crates/propolis-config-toml/src/lib.rs | 14 ++- crates/propolis-config-toml/src/spec.rs | 5 +- lib/propolis/src/enlightenment/bhyve.rs | 66 ------------ lib/propolis/src/enlightenment/bhyve/bits.rs | 32 ++++++ lib/propolis/src/enlightenment/bhyve/mod.rs | 102 +++++++++++++++++++ lib/propolis/src/vmm/machine.rs | 4 +- 9 files changed, 167 insertions(+), 78 deletions(-) delete mode 100644 lib/propolis/src/enlightenment/bhyve.rs create mode 100644 lib/propolis/src/enlightenment/bhyve/bits.rs create mode 100644 lib/propolis/src/enlightenment/bhyve/mod.rs diff --git a/bin/propolis-server/src/lib/vm/ensure.rs b/bin/propolis-server/src/lib/vm/ensure.rs index 3f7d50709..e552f4de3 100644 --- a/bin/propolis-server/src/lib/vm/ensure.rs +++ b/bin/propolis-server/src/lib/vm/ensure.rs @@ -499,7 +499,7 @@ async fn initialize_vm_objects( let guest_hv_interface = match &spec.board.guest_hv_interface { GuestHypervisorInterface::Bhyve => { - Arc::new(BhyveGuestInterface) as Arc + Arc::new(BhyveGuestInterface::new()) as Arc } GuestHypervisorInterface::HyperV { features } => { let mut hv_features = HyperVFeatures::default(); diff --git a/bin/propolis-standalone/src/config.rs b/bin/propolis-standalone/src/config.rs index 6a0d24353..e5a5b915e 100644 --- a/bin/propolis-standalone/src/config.rs +++ b/bin/propolis-standalone/src/config.rs @@ -92,17 +92,25 @@ pub struct Main { /// may influence many aspects of a VM. Most immediately, different interfaces /// have different CPUID leaves, but can also support para-virtualized features /// such as additional hypercalls and MSRs. -#[derive(Clone, Default, Serialize, Deserialize, Debug, PartialEq)] +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)] #[serde(tag = "type", rename_all = "lowercase")] pub enum HypervisorInterface { - #[default] - Bhyve, + Bhyve { + #[serde(default)] + ext_msi_dest_id: bool, + }, HyperV { reference_tsc: bool, }, } +impl Default for HypervisorInterface { + fn default() -> Self { + HypervisorInterface::Bhyve { ext_msi_dest_id: false } + } +} + #[derive(Copy, Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub enum BindingStrategy { diff --git a/bin/propolis-standalone/src/main.rs b/bin/propolis-standalone/src/main.rs index 57c290331..4e638155b 100644 --- a/bin/propolis-standalone/src/main.rs +++ b/bin/propolis-standalone/src/main.rs @@ -811,8 +811,10 @@ fn build_machine( )?; let hv = match hv_interface { - config::HypervisorInterface::Bhyve => { - Arc::new(bhyve::BhyveGuestInterface) as Arc + config::HypervisorInterface::Bhyve { ext_msi_dest_id } => { + let feats = bhyve::Features { ext_msi_dest_id: *ext_msi_dest_id }; + Arc::new(bhyve::BhyveGuestInterface::with_features(feats)?) + as Arc } config::HypervisorInterface::HyperV { reference_tsc } => { let hv_feats = hyperv::Features { reference_tsc: *reference_tsc }; diff --git a/crates/propolis-config-toml/src/lib.rs b/crates/propolis-config-toml/src/lib.rs index f98820ff6..0120282c2 100644 --- a/crates/propolis-config-toml/src/lib.rs +++ b/crates/propolis-config-toml/src/lib.rs @@ -58,17 +58,25 @@ impl Default for Config { /// may influence many aspects of a VM. Most immediately, different interfaces /// have different CPUID leaves, but can also support para-virtualized features /// such as additional hypercalls and MSRs. -#[derive(Clone, Default, Serialize, Deserialize, Debug, PartialEq)] +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)] #[serde(tag = "type", rename_all = "lowercase")] pub enum HypervisorInterface { - #[default] - Bhyve, + Bhyve { + #[serde(default)] + ext_msi_dest_id: bool, + }, HyperV { reference_tsc: bool, }, } +impl Default for HypervisorInterface { + fn default() -> Self { + HypervisorInterface::Bhyve { ext_msi_dest_id: false } + } +} + /// Settings covering the VM "at large". /// /// This corresponds to (and is a subset of) the `main` block understood by diff --git a/crates/propolis-config-toml/src/spec.rs b/crates/propolis-config-toml/src/spec.rs index 34ea865e6..a049d2424 100644 --- a/crates/propolis-config-toml/src/spec.rs +++ b/crates/propolis-config-toml/src/spec.rs @@ -115,7 +115,10 @@ impl TryFrom<&super::Config> for SpecConfig { .hv_interface .as_ref() .map(|hv| match hv { - HypervisorInterface::Bhyve => GuestHypervisorInterface::Bhyve, + HypervisorInterface::Bhyve { ext_msi_dest_id: _ } => { + // XXX(luqman) update api spec version to accept features + GuestHypervisorInterface::Bhyve + } HypervisorInterface::HyperV { reference_tsc } => { let mut features = BTreeSet::new(); if *reference_tsc { diff --git a/lib/propolis/src/enlightenment/bhyve.rs b/lib/propolis/src/enlightenment/bhyve.rs deleted file mode 100644 index 7d0d35d76..000000000 --- a/lib/propolis/src/enlightenment/bhyve.rs +++ /dev/null @@ -1,66 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Provides a bhyve-compatible guest-hypervisor interface. -//! -//! This interface supplies no special enlightenments; it merely identifies -//! itself as a bhyve hypervisor in CPUID leaf 0x4000_0000. - -use std::sync::Arc; - -use cpuid_utils::{ - bits::HYPERVISOR_BASE_LEAF, CpuidIdent, CpuidSet, CpuidValues, -}; - -use crate::{ - accessors::MemAccessor, - common::{Lifecycle, VcpuId}, - enlightenment::{AddCpuidError, Enlightenment}, - msr::{MsrId, RdmsrOutcome, WrmsrOutcome}, - vmm::VmmHdl, -}; - -/// An implementation of the bhyve guest-hypervisor interface. This interface -/// exposes no special enlightenments; its only purpose is to inject the -/// appropriate hypervisor ID into CPUID leaf 0x4000_0000, since this leaf will -/// not otherwise appear in a propolis-server instance specification's CPUID -/// settings. -pub struct BhyveGuestInterface; - -impl Lifecycle for BhyveGuestInterface { - fn type_name(&self) -> &'static str { - "bhyve-guest-interface" - } -} - -impl Enlightenment for BhyveGuestInterface { - fn add_cpuid(&self, cpuid: &mut CpuidSet) -> Result<(), AddCpuidError> { - let mut to_add = CpuidSet::new(cpuid.vendor()); - to_add - .insert( - CpuidIdent::leaf(HYPERVISOR_BASE_LEAF), - // Leaf 0x4000_0000 is the maximum hypervisor leaf. "bhyve bhyve " - // is the vendor ID, split across ebx/ecx/edx. - CpuidValues { - eax: HYPERVISOR_BASE_LEAF, - ebx: 0x76796862, - ecx: 0x68622065, - edx: 0x20657679, - }, - ) - .expect("the map was previously empty"); - - super::add_cpuid(cpuid, to_add) - } - - fn rdmsr(&self, _vcpu: VcpuId, _msr: MsrId) -> RdmsrOutcome { - RdmsrOutcome::NotHandled - } - - fn wrmsr(&self, _vcpu: VcpuId, _msr: MsrId, _value: u64) -> WrmsrOutcome { - WrmsrOutcome::NotHandled - } - - fn attach(&self, _parent: &MemAccessor, _vmm_hdl: Arc) {} -} diff --git a/lib/propolis/src/enlightenment/bhyve/bits.rs b/lib/propolis/src/enlightenment/bhyve/bits.rs new file mode 100644 index 000000000..db85ec4d8 --- /dev/null +++ b/lib/propolis/src/enlightenment/bhyve/bits.rs @@ -0,0 +1,32 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Constant definitions and flags for bhyve enlightenments. + +use cpuid_utils::{CpuidIdent, CpuidValues}; + +/// bhyve hypervisor identifier at CPUID leaf 0x4000_0000. bhyve only supports +/// 1 leaf in the hypervisor range ([`BHYVE_FEATURES_CPUID_LEAF_1`]) which is +/// returned in eax with ebx/ecx/edx containing the vendor ID: "bhyve bhyve ". +pub(super) const BHYVE_LEAF_0_VALUES: CpuidValues = CpuidValues { + eax: BHYVE_FEATURES_CPUID_LEAF_1.leaf, + ebx: 0x76796862, + ecx: 0x68622065, + edx: 0x20657679, +}; + +/// bhyve leaf 0x4000_0001 describes features supported by the hypervisor that +/// a guest may take advantage of in eax. +pub(super) const BHYVE_FEATURES_CPUID_LEAF_1: CpuidIdent = + CpuidIdent { leaf: 0x4000_0001, subleaf: None }; + +bitflags::bitflags! { + /// bhyve leaf 0x4000_0001 eax details the supported set of features. + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub(super) struct BhyveLeaf1Eax: u32 { + const EXT_MSI_DEST_ID = 1 << 0; + + // Bits 1-31 are reserved. + } +} diff --git a/lib/propolis/src/enlightenment/bhyve/mod.rs b/lib/propolis/src/enlightenment/bhyve/mod.rs new file mode 100644 index 000000000..8d1b2afb8 --- /dev/null +++ b/lib/propolis/src/enlightenment/bhyve/mod.rs @@ -0,0 +1,102 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Provides a bhyve-compatible guest-hypervisor interface. +//! +//! This interface indentifies itself as a bhyve hypervisor in CPUID leaf +//! 0x4000_0000 and optional enlightenments. + +use std::sync::Arc; + +use cpuid_utils::{ + bits::HYPERVISOR_BASE_LEAF, CpuidIdent, CpuidSet, CpuidValues, +}; + +use crate::{ + accessors::MemAccessor, + common::{Lifecycle, VcpuId}, + enlightenment::{ + bhyve::bits::{ + BhyveLeaf1Eax, BHYVE_FEATURES_CPUID_LEAF_1, BHYVE_LEAF_0_VALUES, + }, + AddCpuidError, Enlightenment, + }, + msr::{MsrId, RdmsrOutcome, WrmsrOutcome}, + vmm::VmmHdl, +}; + +mod bits; + +/// A set of features that can be enabled for a given bhyve enlightenment stack. +#[derive(Clone, Copy, Debug, Default)] +pub struct Features { + /// Enable support for 15-bit Extended MSI Destination IDs + pub ext_msi_dest_id: bool, +} + +/// An implementation of the bhyve guest-hypervisor interface. In addition to +/// any optional enlightenments that are enabled, this interface also serves to +/// inject the appropriate hypervisor ID into CPUID leaf 0x4000_0000, since this +/// leaf will not otherwise appear in a propolis-server instance specification's +/// CPUID settings. +pub struct BhyveGuestInterface { + features: Features, +} + +impl BhyveGuestInterface { + /// Returns a basic enlightenment stack with just the hypervisor ID. + pub fn new() -> Self { + Self { features: Default::default() } + } + + /// Returns an enlightenment stack with the given set of features. Certain + /// features may require support from the kernel vmm component and so this + /// may fail if running on a version without the required capabilities. + pub fn with_features(features: Features) -> std::io::Result { + if features.ext_msi_dest_id { + if bhyve_api::api_version()? < bhyve_api::ApiVersion::V21 { + return Err(std::io::ErrorKind::Unsupported.into()); + } + } + Ok(Self { features }) + } +} + +impl Lifecycle for BhyveGuestInterface { + fn type_name(&self) -> &'static str { + "bhyve-guest-interface" + } +} + +impl Enlightenment for BhyveGuestInterface { + fn add_cpuid(&self, cpuid: &mut CpuidSet) -> Result<(), AddCpuidError> { + let mut to_add = CpuidSet::new(cpuid.vendor()); + to_add + .insert(CpuidIdent::leaf(HYPERVISOR_BASE_LEAF), BHYVE_LEAF_0_VALUES) + .expect("the map was previously empty"); + + let mut leaf_1_eax = BhyveLeaf1Eax::default(); + if self.features.ext_msi_dest_id { + leaf_1_eax |= BhyveLeaf1Eax::EXT_MSI_DEST_ID; + } + to_add + .insert( + BHYVE_FEATURES_CPUID_LEAF_1, + CpuidValues { eax: leaf_1_eax.bits(), ..Default::default() }, + ) + .expect("bhyve CPUID values don't conflict"); + + super::add_cpuid(cpuid, to_add) + } + + fn rdmsr(&self, _vcpu: VcpuId, _msr: MsrId) -> RdmsrOutcome { + RdmsrOutcome::NotHandled + } + + fn wrmsr(&self, _vcpu: VcpuId, _msr: MsrId, _value: u64) -> WrmsrOutcome { + WrmsrOutcome::NotHandled + } + + fn attach(&self, _parent: &MemAccessor, _vmm_hdl: Arc) {} +} diff --git a/lib/propolis/src/vmm/machine.rs b/lib/propolis/src/vmm/machine.rs index 19a9b3970..8a7492d1c 100644 --- a/lib/propolis/src/vmm/machine.rs +++ b/lib/propolis/src/vmm/machine.rs @@ -125,7 +125,7 @@ impl Machine { let bus_mmio = Arc::new(MmioBus::new(MAX_PHYSMEM)); let bus_pio = Arc::new(PioBus::new()); - let guest_hv_interface = Arc::new(BhyveGuestInterface); + let guest_hv_interface = Arc::new(BhyveGuestInterface::new()); let vcpus = vec![Vcpu::new( hdl.clone(), @@ -273,7 +273,7 @@ impl Builder { let guest_hv_interface = self .guest_hv_interface .take() - .unwrap_or(Arc::new(BhyveGuestInterface)); + .unwrap_or(Arc::new(BhyveGuestInterface::new())); guest_hv_interface.attach(&acc_mem, hdl.clone()); let vcpus = (0..self.max_cpu) From cfd08d45de2d7f917e7ee320f7808c688d4e3490 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Tue, 17 Nov 2026 16:08:43 -0800 Subject: [PATCH 4/8] Update hyper-v enlightenment stack with EXTENDED_IOAPIC_RTE --- bin/propolis-server/src/lib/vm/ensure.rs | 2 +- bin/propolis-standalone/src/config.rs | 2 + bin/propolis-standalone/src/main.rs | 12 ++++-- crates/propolis-config-toml/src/lib.rs | 2 + crates/propolis-config-toml/src/spec.rs | 7 +++- lib/propolis/src/enlightenment/hyperv/bits.rs | 36 ++++++++++++++++- lib/propolis/src/enlightenment/hyperv/mod.rs | 39 +++++++++++++++++-- 7 files changed, 90 insertions(+), 10 deletions(-) diff --git a/bin/propolis-server/src/lib/vm/ensure.rs b/bin/propolis-server/src/lib/vm/ensure.rs index e552f4de3..8979791ff 100644 --- a/bin/propolis-server/src/lib/vm/ensure.rs +++ b/bin/propolis-server/src/lib/vm/ensure.rs @@ -511,7 +511,7 @@ async fn initialize_vm_objects( } } - let hv = HyperV::new(&vmm_log, hv_features); + let hv = HyperV::new(&vmm_log, hv_features)?; Arc::new(hv) as Arc } }; diff --git a/bin/propolis-standalone/src/config.rs b/bin/propolis-standalone/src/config.rs index e5a5b915e..9c1182427 100644 --- a/bin/propolis-standalone/src/config.rs +++ b/bin/propolis-standalone/src/config.rs @@ -102,6 +102,8 @@ pub enum HypervisorInterface { HyperV { reference_tsc: bool, + #[serde(default)] + extended_ioapic_rte: bool, }, } diff --git a/bin/propolis-standalone/src/main.rs b/bin/propolis-standalone/src/main.rs index 4e638155b..eb699fd29 100644 --- a/bin/propolis-standalone/src/main.rs +++ b/bin/propolis-standalone/src/main.rs @@ -816,9 +816,15 @@ fn build_machine( Arc::new(bhyve::BhyveGuestInterface::with_features(feats)?) as Arc } - config::HypervisorInterface::HyperV { reference_tsc } => { - let hv_feats = hyperv::Features { reference_tsc: *reference_tsc }; - Arc::new(hyperv::HyperV::new(log, hv_feats)) + config::HypervisorInterface::HyperV { + reference_tsc, + extended_ioapic_rte, + } => { + let hv_feats = hyperv::Features { + reference_tsc: *reference_tsc, + extended_ioapic_rte: *extended_ioapic_rte, + }; + Arc::new(hyperv::HyperV::new(log, hv_feats)?) as Arc } }; diff --git a/crates/propolis-config-toml/src/lib.rs b/crates/propolis-config-toml/src/lib.rs index 0120282c2..599389662 100644 --- a/crates/propolis-config-toml/src/lib.rs +++ b/crates/propolis-config-toml/src/lib.rs @@ -68,6 +68,8 @@ pub enum HypervisorInterface { HyperV { reference_tsc: bool, + #[serde(default)] + extended_ioapic_rte: bool, }, } diff --git a/crates/propolis-config-toml/src/spec.rs b/crates/propolis-config-toml/src/spec.rs index a049d2424..9a254c8b0 100644 --- a/crates/propolis-config-toml/src/spec.rs +++ b/crates/propolis-config-toml/src/spec.rs @@ -119,7 +119,12 @@ impl TryFrom<&super::Config> for SpecConfig { // XXX(luqman) update api spec version to accept features GuestHypervisorInterface::Bhyve } - HypervisorInterface::HyperV { reference_tsc } => { + HypervisorInterface::HyperV { + reference_tsc, + extended_ioapic_rte: _, + } => { + // XXX(luqman) update api spec version to accept + // extended_ioapic_rte let mut features = BTreeSet::new(); if *reference_tsc { features.insert(HyperVFeatureFlag::ReferenceTsc); diff --git a/lib/propolis/src/enlightenment/hyperv/bits.rs b/lib/propolis/src/enlightenment/hyperv/bits.rs index a7ba860d8..2b181de12 100644 --- a/lib/propolis/src/enlightenment/hyperv/bits.rs +++ b/lib/propolis/src/enlightenment/hyperv/bits.rs @@ -9,7 +9,7 @@ //! Where possible, constants in this module (such as MSR identifiers) are given //! names that match those used in the TLFS. -use cpuid_utils::CpuidValues; +use cpuid_utils::{CpuidIdent, CpuidValues}; /// Hyper-V-compatible hypervisors are required to support hypervisor CPUID /// leaves up to 0x4000_0005. @@ -53,7 +53,7 @@ pub(super) fn hyperv_leaf_0_values(max_leaf: u32) -> CpuidValues { /// Hyper-V leaf 0x4000_0001 contains an (ostensibly vendor-neutral) interface /// identifier. eax receives "Hv#1"; the other three outputs are reserved. pub(super) const HYPERV_LEAF_1_VALUES: CpuidValues = - CpuidValues { eax: 0x31237648, ebx: 0, ecx: 0, edx: 0 }; + CpuidValues { eax: u32::from_le_bytes(*b"Hv#1"), ebx: 0, ecx: 0, edx: 0 }; /// Hyper-V leaf 0x4000_0002 contains hypervisor version information. To avoid /// having to reason about what it means to expose a specific hypervisor version @@ -98,6 +98,38 @@ pub(super) const HYPERV_LEAF_4_VALUES: CpuidValues = pub(super) const HYPERV_LEAF_5_VALUES: CpuidValues = CpuidValues { eax: 0, ebx: 0, ecx: 0, edx: 0 }; +/// Hyper-V leaf 0x4000_0081 is undocumented in the TLFS but referenced +/// publicy in various open-source, Hyper-V-aware guests like Linux. +/// Like 0x4000_0001, it returns an interface identifier ("VS#1") in eax +/// and indicates the presence of leaf 0x4000_0082. +pub(super) const HYPERV_VIRT_STACK_INTERFACE_LEAF: CpuidIdent = + CpuidIdent { leaf: 0x4000_0081, subleaf: None }; + +/// Hyper-V leaf 0x4000_0081 carries the Hyper-V "Virtualization Stack" +/// identifier in eax ("VS#1"), with ebx/ecx/edx set to 0. In contrast to +/// Hyper-V leaf 0x4000_0001, this leaf seems to relate to the user-mode +/// portion of the virtualization stack rather than the core hypervisor. +pub(super) const HYPERV_VIRT_STACK_INTERFACE_LEAF_VALUES: CpuidValues = + CpuidValues { eax: u32::from_le_bytes(*b"VS#1"), ebx: 0, ecx: 0, edx: 0 }; + +/// Hyper-V leaf 0x4000_0082 contains a set of bitflags indicating features +/// exposed by the hypervisor and thus available to a guest. Outside of +/// certain bits referenced publicly, this leaf is undocumented and any +/// other bits are reserved and always set to 0. +pub(super) const HYPERV_VIRT_STACK_PROPERTIES_LEAF: CpuidIdent = + CpuidIdent { leaf: 0x4000_0082, subleaf: None }; + +bitflags::bitflags! { + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub(super) struct HyperVLeaf82Eax: u32 { + /// This bit indicates support for 15-bit Extended Destination (APIC) + /// ID in I/O APIC Redirection Table Entries (RTEs) and MSIs. + /// For guests with support, this enables more than 255 vCPUs + /// without Interrupt Remapping/vIOMMU. + const EXTENDED_IOAPIC_RTE = 1 << 2; + } +} + /// Allows the guest to report its type and version information. See TLFS /// section 2.6 for details about this MSR's format. /// diff --git a/lib/propolis/src/enlightenment/hyperv/mod.rs b/lib/propolis/src/enlightenment/hyperv/mod.rs index fcaa7f144..123970b1a 100644 --- a/lib/propolis/src/enlightenment/hyperv/mod.rs +++ b/lib/propolis/src/enlightenment/hyperv/mod.rs @@ -61,6 +61,10 @@ const TYPE_NAME: &str = "hyperv-guest-interface"; pub struct Features { /// Enables the reference time MSR and the reference TSC page. pub reference_tsc: bool, + + /// Enables support for 15-bit Extended Destination IDs in I/O APIC + /// Redirection Table Entries and MSIs. + pub extended_ioapic_rte: bool, } /// Wrapper around a hypercall overlay page. @@ -194,8 +198,21 @@ impl HyperV { /// enlightenment stack before starting any VM components that depend on it. /// Otherwise the stack may panic while the VM is running. /// + /// Certain features may require support from the kernel vmm component, so + /// this may fail if running on a version without the required + /// capabilities. + /// /// [`attach`]: super::Enlightenment::attach - pub fn new(log: &slog::Logger, features: Features) -> Self { + pub fn new( + log: &slog::Logger, + features: Features, + ) -> std::io::Result { + if features.extended_ioapic_rte + && bhyve_api::api_version()? < bhyve_api::ApiVersion::V21 + { + return Err(std::io::ErrorKind::Unsupported.into()); + } + let acc_mem = MemAccessor::new_orphan(); let log = log.new(slog::o!("component" => "hyperv")); info!( @@ -204,13 +221,13 @@ impl HyperV { "features" => ?features ); - Self { + Ok(Self { log, features, inner: Mutex::new(Inner::new(&features)), acc_mem, vmm_hdl: OnceLock::new(), - } + }) } /// Returns a reference to this manager's VMM handle. @@ -416,6 +433,22 @@ impl super::Enlightenment for HyperV { add_to_set(CpuidIdent::leaf(0x4000_0004), HYPERV_LEAF_4_VALUES); add_to_set(CpuidIdent::leaf(0x4000_0005), HYPERV_LEAF_5_VALUES); + let mut leaf_82_eax = HyperVLeaf82Eax::default(); + if self.features.extended_ioapic_rte { + leaf_82_eax |= HyperVLeaf82Eax::EXTENDED_IOAPIC_RTE; + } + + if !leaf_82_eax.is_empty() { + add_to_set( + HYPERV_VIRT_STACK_INTERFACE_LEAF, + HYPERV_VIRT_STACK_INTERFACE_LEAF_VALUES, + ); + add_to_set( + HYPERV_VIRT_STACK_PROPERTIES_LEAF, + CpuidValues { eax: leaf_82_eax.bits(), ..Default::default() }, + ); + } + // Set the maximum available CPUID leaf to the smallest value required // to expose all of the enlightenment's features. // From dcb54ae018577711c885b7d04fa88cd8fe5ae0b9 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Tue, 17 Nov 2026 21:40:39 -0800 Subject: [PATCH 5/8] propolis-server: add new api version for ext-msi-dest-id enlightenments --- bin/propolis-server/src/lib/migrate/types.rs | 57 +++- .../src/lib/spec/api_spec_v1.rs | 19 +- .../src/lib/spec/api_spec_v2.rs | 5 +- .../src/lib/spec/api_spec_v3.rs | 5 +- .../src/lib/spec/api_spec_v6.rs | 34 +- .../src/lib/spec/api_spec_v8.rs | 153 +++++++++ bin/propolis-server/src/lib/spec/builder.rs | 2 +- bin/propolis-server/src/lib/spec/mod.rs | 5 +- bin/propolis-server/src/lib/vm/ensure.rs | 20 +- .../src/ext_msi_dest_id/api.rs | 64 ++++ .../src/ext_msi_dest_id/components/board.rs | 307 ++++++++++++++++++ .../src/ext_msi_dest_id/components/mod.rs | 5 + .../src/ext_msi_dest_id/instance_spec.rs | 80 +++++ .../src/ext_msi_dest_id/mod.rs | 13 + .../propolis-api-types-versions/src/latest.rs | 19 +- crates/propolis-api-types-versions/src/lib.rs | 2 + crates/propolis-config-toml/src/spec.rs | 27 +- crates/propolis-server-api/src/lib.rs | 60 +++- .../propolis-server-7.0.0-712b68.json.gitstub | 1 + ...json => propolis-server-8.0.0-afac14.json} | 55 +++- .../propolis-server-latest.json | 2 +- 21 files changed, 866 insertions(+), 69 deletions(-) create mode 100644 bin/propolis-server/src/lib/spec/api_spec_v8.rs create mode 100644 crates/propolis-api-types-versions/src/ext_msi_dest_id/api.rs create mode 100644 crates/propolis-api-types-versions/src/ext_msi_dest_id/components/board.rs create mode 100644 crates/propolis-api-types-versions/src/ext_msi_dest_id/components/mod.rs create mode 100644 crates/propolis-api-types-versions/src/ext_msi_dest_id/instance_spec.rs create mode 100644 crates/propolis-api-types-versions/src/ext_msi_dest_id/mod.rs create mode 100644 openapi/propolis-server/propolis-server-7.0.0-712b68.json.gitstub rename openapi/propolis-server/{propolis-server-7.0.0-712b68.json => propolis-server-8.0.0-afac14.json} (97%) diff --git a/bin/propolis-server/src/lib/migrate/types.rs b/bin/propolis-server/src/lib/migrate/types.rs index e8688b14c..3ccb9e005 100644 --- a/bin/propolis-server/src/lib/migrate/types.rs +++ b/bin/propolis-server/src/lib/migrate/types.rs @@ -89,12 +89,14 @@ use serde::{Deserialize, Serialize}; use propolis_api_types::instance::ReplacementComponent; -use propolis_api_types_versions::{v1, v2, v3, v6}; +use propolis_api_types_versions::{v1, v2, v3, v6, v8}; use std::collections::BTreeMap; use crate::migrate::MigrateError; -use crate::spec::{api_spec_v1, api_spec_v2, api_spec_v3, api_spec_v6, Spec}; +use crate::spec::{ + api_spec_v1, api_spec_v2, api_spec_v3, api_spec_v6, api_spec_v8, Spec, +}; /// A wrapper for one of any supported `InstanceSpec` that describe a /// to-be-migrated VM. @@ -113,6 +115,7 @@ pub(crate) enum VersionedInstanceSpec { V2(v2::instance_spec::InstanceSpec), V3(v3::instance_spec::InstanceSpec), V6(v6::instance_spec::InstanceSpec), + V8(v8::instance_spec::InstanceSpec), } impl VersionedInstanceSpec { @@ -142,9 +145,13 @@ impl VersionedInstanceSpec { TryInto::::try_into(spec.clone()) { VersionedInstanceSpec::V3(v3_spec) + } else if let Ok(v6_spec) = + TryInto::::try_into(spec.clone()) + { + VersionedInstanceSpec::V6(v6_spec) } else { - VersionedInstanceSpec::V6( - Into::::into(spec.clone()), + VersionedInstanceSpec::V8( + Into::::into(spec.clone()), ) }; Ok(versioned) @@ -186,8 +193,50 @@ impl VersionedInstanceSpec { .map_err(|e| MigrateError::PreambleParse(e.to_string()))? .finish() } + VersionedInstanceSpec::V8(mut source_spec) => { + api_spec_v8::amend(&mut source_spec, replacements)?; + + api_spec_v8::v8_to_spec_builder(source_spec) + .map_err(|e| MigrateError::PreambleParse(e.to_string()))? + .finish() + } }; Ok(amended_spec) } } + +#[cfg(test)] +mod test { + use propolis_api_types_versions::latest::components::board::GuestHypervisorInterface; + + use super::VersionedInstanceSpec; + use crate::spec::api_spec_v8::test::{ + spec_with_interface, v8_only_interfaces, + }; + + #[test] + fn default_interface_uses_oldest_version() { + let spec = spec_with_interface(GuestHypervisorInterface::default()); + assert!(matches!( + VersionedInstanceSpec::from_spec(&spec).unwrap(), + VersionedInstanceSpec::V1(_) + )); + } + + #[test] + fn v8_only_interfaces_migrate_as_v8() { + for iface in v8_only_interfaces() { + let spec = spec_with_interface(iface.clone()); + let versioned = VersionedInstanceSpec::from_spec(&spec).unwrap(); + let VersionedInstanceSpec::V8(ref v8_spec) = versioned else { + panic!("expected a v8 spec for interface {iface:?}"); + }; + assert_eq!(v8_spec.board.guest_hv_interface, iface); + + let roundtrip = + versioned.into_amended_spec(&Default::default()).unwrap(); + assert_eq!(roundtrip.board.guest_hv_interface, iface); + } + } +} diff --git a/bin/propolis-server/src/lib/spec/api_spec_v1.rs b/bin/propolis-server/src/lib/spec/api_spec_v1.rs index 207096277..19d336e66 100644 --- a/bin/propolis-server/src/lib/spec/api_spec_v1.rs +++ b/bin/propolis-server/src/lib/spec/api_spec_v1.rs @@ -10,13 +10,10 @@ use std::collections::BTreeMap; use propolis_api_types::instance::ReplacementComponent; use propolis_api_types::instance_spec::{ - components::{ - board::Board as InstanceSpecBoard, - devices::{BootSettings, SerialPort as SerialPortDesc}, - }, + components::devices::{BootSettings, SerialPort as SerialPortDesc}, SpecKey, }; -use propolis_api_types_versions::{v1, v2, v3, v6}; +use propolis_api_types_versions::{v1, v2, v3, v6, v8}; #[cfg(feature = "falcon")] use propolis_api_types::instance_spec::components::devices::SoftNpuPort as SoftNpuPortSpec; @@ -136,11 +133,19 @@ impl TryFrom for v1::instance_spec::InstanceSpec { spec.components.insert(key, val); } - let board = InstanceSpecBoard { + // Guest hypervisor interface features added in v8 can't be expressed + // in a v1 board. + let guest_hv_interface = board.guest_hv_interface.try_into().map_err( + |e: v8::components::board::InvalidV1GuestHypervisorInterface| { + LegacyApiSpecError::IncompatibleComponent(e.to_string()) + }, + )?; + + let board = v1::components::board::Board { cpus: board.cpus, memory_mb: board.memory_mb, chipset: board.chipset, - guest_hv_interface: board.guest_hv_interface, + guest_hv_interface, cpuid: Some(cpuid.into_instance_spec_cpuid()), }; let mut spec = v1::instance_spec::InstanceSpec { diff --git a/bin/propolis-server/src/lib/spec/api_spec_v2.rs b/bin/propolis-server/src/lib/spec/api_spec_v2.rs index f01396586..946d784dd 100644 --- a/bin/propolis-server/src/lib/spec/api_spec_v2.rs +++ b/bin/propolis-server/src/lib/spec/api_spec_v2.rs @@ -98,7 +98,7 @@ mod test { cpus: 4, memory_mb: 512, chipset: Chipset::I440Fx(I440Fx { enable_pcie: false }), - guest_hv_interface: GuestHypervisorInterface::Bhyve, + guest_hv_interface: GuestHypervisorInterface::default(), // Providing *any* CPUID settings keeps Propolis from querying // bhyve for defaults to use instead, which requires .. byhve // *and* VMM access. Provide a (useless) empty CPUID set so this @@ -122,7 +122,8 @@ mod test { .expect("SMBIOS type 1 input preserved"); assert_eq!(&smbios_input, smbios); - let v6_spec = v6::instance_spec::InstanceSpec::from(spec.clone()); + let v6_spec = + v6::instance_spec::InstanceSpec::try_from(spec.clone()).unwrap(); let smbios = v6_spec.smbios.as_ref().expect("SMBIOS type 1 input preserved"); assert_eq!(&smbios_input, smbios); diff --git a/bin/propolis-server/src/lib/spec/api_spec_v3.rs b/bin/propolis-server/src/lib/spec/api_spec_v3.rs index 146974c9d..f64df2386 100644 --- a/bin/propolis-server/src/lib/spec/api_spec_v3.rs +++ b/bin/propolis-server/src/lib/spec/api_spec_v3.rs @@ -144,7 +144,7 @@ mod test { cpus: 4, memory_mb: 512, chipset: Chipset::I440Fx(I440Fx { enable_pcie: false }), - guest_hv_interface: GuestHypervisorInterface::Bhyve, + guest_hv_interface: GuestHypervisorInterface::default(), // Providing *any* CPUID settings keeps Propolis from querying // bhyve for defaults to use instead, which requires .. byhve // *and* VMM access. Provide a (useless) empty CPUID set so this @@ -179,7 +179,8 @@ mod test { .finish(); assert!(spec.vsock.is_some()); - let v6_spec = v6::instance_spec::InstanceSpec::from(spec.clone()); + let v6_spec = + v6::instance_spec::InstanceSpec::try_from(spec.clone()).unwrap(); let v6_comp: v6::instance_spec::Component = vsock_comp.clone(); assert_eq!(v6_spec.components.get(&vsock_id), Some(&v6_comp)); diff --git a/bin/propolis-server/src/lib/spec/api_spec_v6.rs b/bin/propolis-server/src/lib/spec/api_spec_v6.rs index 40364ff71..19cc36c5a 100644 --- a/bin/propolis-server/src/lib/spec/api_spec_v6.rs +++ b/bin/propolis-server/src/lib/spec/api_spec_v6.rs @@ -9,14 +9,19 @@ use std::collections::BTreeMap; use propolis_api_types::instance::ReplacementComponent; use propolis_api_types::instance_spec::SpecKey; -use propolis_api_types_versions::{v3, v6}; +use propolis_api_types_versions::{v3, v6, v8}; -use super::{builder::SpecBuilder, ApiSpecError, Disk, Spec, StorageDevice}; +use super::{ + builder::SpecBuilder, ApiSpecError, Disk, LegacyApiSpecError, Spec, + StorageDevice, +}; use crate::migrate::MigrateError; use crate::spec::api_spec_latest; -impl From for v6::instance_spec::InstanceSpec { - fn from(mut val: Spec) -> Self { +impl TryFrom for v6::instance_spec::InstanceSpec { + type Error = LegacyApiSpecError; + + fn try_from(mut val: Spec) -> Result { // v6 adds a new field on NvmeDisk. Such disks probably can't be // converted to v3 components and would cause a conversion from // Spec->v3::instance_spec::InstanceSpec to fail. So, extract those @@ -36,15 +41,7 @@ impl From for v6::instance_spec::InstanceSpec { } val.disks.retain(|_, disk| !v6_only_disk(disk)); - let v3_spec: v3::instance_spec::InstanceSpec = - val.try_into().unwrap_or_else(|e| { - unreachable!( - "Converting to Spec without v6 bits to v3 failed: {e}. \ - This is currently impossible. When Spec to \ - v6::instance_spec::InstanceSpec becomes fallible, \ - this should `?`." - ); - }); + let v3_spec: v3::instance_spec::InstanceSpec = val.try_into()?; let mut spec: v6::instance_spec::InstanceSpec = v3_spec.into(); @@ -78,7 +75,7 @@ impl From for v6::instance_spec::InstanceSpec { insert_component(&mut spec, backend_id, backend_component); } - spec + Ok(spec) } } @@ -88,10 +85,15 @@ impl From for v6::instance_spec::InstanceSpec { pub(crate) fn v6_to_spec_builder( value: v6::instance_spec::InstanceSpec, ) -> Result { - api_spec_latest::latest_to_spec_builder(value) + // Converting v6 to v8 is lossless, so just do that and piggyback on the + // latest `InstanceSpec->SpecBuilder`. + let v8_spec: v8::instance_spec::InstanceSpec = value.into(); + api_spec_latest::latest_to_spec_builder(v8_spec) } -fn amend_component( +// v8 instance specs use the same `Component` type as v6, so this is also used +// to amend v8 specs. +pub(crate) fn amend_component( id: &SpecKey, to_amend: &mut v6::instance_spec::Component, replacement: &ReplacementComponent, diff --git a/bin/propolis-server/src/lib/spec/api_spec_v8.rs b/bin/propolis-server/src/lib/spec/api_spec_v8.rs new file mode 100644 index 000000000..ab16c770b --- /dev/null +++ b/bin/propolis-server/src/lib/spec/api_spec_v8.rs @@ -0,0 +1,153 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Conversions between [`propolis_api_types_versions::v8`] instance specs and +//! the internal [`super::Spec`] representation. + +use std::collections::BTreeMap; + +use propolis_api_types::instance::ReplacementComponent; +use propolis_api_types::instance_spec::SpecKey; +use propolis_api_types_versions::{v6, v8}; + +use super::{api_spec_latest, builder::SpecBuilder, ApiSpecError, Spec}; +use crate::migrate::MigrateError; + +impl From for v8::instance_spec::InstanceSpec { + fn from(mut val: Spec) -> Self { + // v8 only changes the guest hypervisor interface, allowing it to carry + // features that can't be expressed in v6. Swap in the default + // interface (which every version can express), convert the rest of + // the spec to v6, and then put the real interface back. + let guest_hv_interface = + std::mem::take(&mut val.board.guest_hv_interface); + + let v6_spec: v6::instance_spec::InstanceSpec = + val.try_into().unwrap_or_else(|e| { + unreachable!( + "Converting Spec without v8 bits to v6 failed: {e}. \ + This is currently impossible. When Spec to \ + v8::instance_spec::InstanceSpec becomes fallible, \ + this should `?`." + ); + }); + + let mut spec: v8::instance_spec::InstanceSpec = v6_spec.into(); + spec.board.guest_hv_interface = guest_hv_interface; + spec + } +} + +/// Parses a v8 instance spec into a [`SpecBuilder`], validating component +/// names, PCI paths, and backend references along the way. Callers can add +/// additional (non-v8) components to the builder before calling `finish()`. +pub(crate) fn v8_to_spec_builder( + value: v8::instance_spec::InstanceSpec, +) -> Result { + api_spec_latest::latest_to_spec_builder(value) +} + +pub(crate) fn amend( + spec: &mut v8::instance_spec::InstanceSpec, + replacements: &BTreeMap, +) -> Result<(), MigrateError> { + for (id, replacement) in replacements { + let Some(to_amend) = spec.components.get_mut(id) else { + return Err(MigrateError::InstanceSpecsIncompatible(format!( + "replacement component {id} not in source spec", + ))); + }; + + super::api_spec_v6::amend_component(id, to_amend, replacement)?; + } + + Ok(()) +} + +#[cfg(test)] +pub(crate) mod test { + use propolis_api_types_versions::latest::components::board::{ + BhyveFeatureFlag, Board, Chipset, Cpuid, GuestHypervisorInterface, + HyperVFeatureFlag, I440Fx, + }; + use propolis_api_types_versions::latest::{ + self, instance_spec::CpuidVendor, + }; + use propolis_api_types_versions::{v1, v6, v8}; + + /// Builds an internal spec with no components and the supplied guest + /// hypervisor interface. + pub(crate) fn spec_with_interface( + guest_hv_interface: GuestHypervisorInterface, + ) -> crate::spec::Spec { + let api_spec = latest::instance_spec::InstanceSpec { + board: Board { + cpus: 4, + memory_mb: 512, + chipset: Chipset::I440Fx(I440Fx { enable_pcie: false }), + guest_hv_interface, + // Providing *any* CPUID settings keeps Propolis from querying + // bhyve for defaults to use instead, which requires bhyve + // *and* VMM access. Provide a (useless) empty CPUID set so this + // test can run on non-illumos test systems. + cpuid: Some(Cpuid { + entries: vec![], + vendor: CpuidVendor::Amd, + }), + }, + components: Default::default(), + smbios: None, + }; + + crate::spec::api_spec_latest::latest_to_spec_builder(api_spec) + .unwrap() + .finish() + } + + /// Guest hypervisor interfaces that can only be expressed in v8 and later. + pub(crate) fn v8_only_interfaces( + ) -> impl IntoIterator { + [ + GuestHypervisorInterface::Bhyve { + features: [BhyveFeatureFlag::ExtMsiDestId].into(), + }, + GuestHypervisorInterface::HyperV { + features: [ + HyperVFeatureFlag::ReferenceTsc, + HyperVFeatureFlag::ExtendedIoapicRte, + ] + .into(), + }, + ] + } + + #[test] + fn reference_tsc_v1() { + let v1_hv = GuestHypervisorInterface::HyperV { + features: [HyperVFeatureFlag::ReferenceTsc].into(), + }; + let spec = spec_with_interface(v1_hv.clone()); + let v1_spec = v1::instance_spec::InstanceSpec::try_from(spec).unwrap(); + assert_eq!( + v8::components::board::GuestHypervisorInterface::from( + v1_spec.board.guest_hv_interface + ), + v1_hv + ); + } + + #[test] + fn v8_features_require_v8() { + for iface in v8_only_interfaces() { + let spec = spec_with_interface(iface.clone()); + assert!(v6::instance_spec::InstanceSpec::try_from(spec.clone()) + .is_err()); + assert!(v1::instance_spec::InstanceSpec::try_from(spec.clone()) + .is_err()); + + let v8_spec = v8::instance_spec::InstanceSpec::from(spec); + assert_eq!(v8_spec.board.guest_hv_interface, iface); + } + } +} diff --git a/bin/propolis-server/src/lib/spec/builder.rs b/bin/propolis-server/src/lib/spec/builder.rs index e3290369e..5116871a1 100644 --- a/bin/propolis-server/src/lib/spec/builder.rs +++ b/bin/propolis-server/src/lib/spec/builder.rs @@ -420,7 +420,7 @@ mod test { cpus: 4, memory_mb: 512, chipset: Chipset::I440Fx(I440Fx { enable_pcie: false }), - guest_hv_interface: GuestHypervisorInterface::Bhyve, + guest_hv_interface: GuestHypervisorInterface::default(), acpi_variant: propolis::firmware::acpi::AcpiVariant::V0, }; diff --git a/bin/propolis-server/src/lib/spec/mod.rs b/bin/propolis-server/src/lib/spec/mod.rs index 79c613650..1aa34d5d2 100644 --- a/bin/propolis-server/src/lib/spec/mod.rs +++ b/bin/propolis-server/src/lib/spec/mod.rs @@ -70,6 +70,7 @@ pub(crate) mod api_spec_v1; pub(crate) mod api_spec_v2; pub(crate) mod api_spec_v3; pub(crate) mod api_spec_v6; +pub(crate) mod api_spec_v8; pub(crate) mod builder; /// An error that can arise in converting @@ -127,7 +128,7 @@ impl TryFrom for Spec { type Error = ApiSpecError; fn try_from(value: InstanceSpec) -> Result { - Ok(api_spec_v6::v6_to_spec_builder(value)?.finish()) + Ok(api_spec_v8::v8_to_spec_builder(value)?.finish()) } } @@ -213,7 +214,7 @@ impl Default for Board { cpus: 0, memory_mb: 0, chipset: Chipset::I440Fx(I440Fx { enable_pcie: false }), - guest_hv_interface: GuestHypervisorInterface::Bhyve, + guest_hv_interface: GuestHypervisorInterface::default(), acpi_variant: AcpiVariant::V0, } } diff --git a/bin/propolis-server/src/lib/vm/ensure.rs b/bin/propolis-server/src/lib/vm/ensure.rs index 8979791ff..8481d1f37 100644 --- a/bin/propolis-server/src/lib/vm/ensure.rs +++ b/bin/propolis-server/src/lib/vm/ensure.rs @@ -97,6 +97,7 @@ use propolis::common::DeviceMetadataMap; use propolis::enlightenment::{ self, bhyve::BhyveGuestInterface, + bhyve::Features as BhyveFeatures, hyperv::{Features as HyperVFeatures, HyperV}, Enlightenment, }; @@ -104,7 +105,7 @@ use propolis_api_types::instance::{ InstanceEnsureResponse, InstanceProperties, InstanceState, }; use propolis_api_types::instance_spec::components::board::{ - GuestHypervisorInterface, HyperVFeatureFlag, + BhyveFeatureFlag, GuestHypervisorInterface, HyperVFeatureFlag, }; use propolis_api_types::migration::InstanceMigrateInitiateResponse; use slog::{debug, info}; @@ -498,8 +499,18 @@ async fn initialize_vm_objects( let vmm_log = log.new(slog::o!("component" => "vmm")); let guest_hv_interface = match &spec.board.guest_hv_interface { - GuestHypervisorInterface::Bhyve => { - Arc::new(BhyveGuestInterface::new()) as Arc + GuestHypervisorInterface::Bhyve { features } => { + let mut bhyve_features = BhyveFeatures::default(); + for f in features { + match f { + BhyveFeatureFlag::ExtMsiDestId => { + bhyve_features.ext_msi_dest_id = true + } + } + } + + let bhyve = BhyveGuestInterface::with_features(bhyve_features)?; + Arc::new(bhyve) as Arc } GuestHypervisorInterface::HyperV { features } => { let mut hv_features = HyperVFeatures::default(); @@ -508,6 +519,9 @@ async fn initialize_vm_objects( HyperVFeatureFlag::ReferenceTsc => { hv_features.reference_tsc = true } + HyperVFeatureFlag::ExtendedIoapicRte => { + hv_features.extended_ioapic_rte = true + } } } diff --git a/crates/propolis-api-types-versions/src/ext_msi_dest_id/api.rs b/crates/propolis-api-types-versions/src/ext_msi_dest_id/api.rs new file mode 100644 index 000000000..c09528402 --- /dev/null +++ b/crates/propolis-api-types-versions/src/ext_msi_dest_id/api.rs @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! API request and response types for the EXT_MSI_DEST_ID API version. + +use std::{collections::BTreeMap, net::SocketAddr}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::instance_spec::InstanceSpec; +use crate::v1::instance::InstanceProperties; +use crate::v1::instance_spec::SpecKey; +use crate::v7; +use crate::v7::instance::ReplacementComponent; + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(tag = "method", content = "value", rename_all = "snake_case")] +pub enum InstanceInitializationMethod { + Spec { + spec: InstanceSpec, + }, + MigrationTarget { + migration_id: Uuid, + src_addr: SocketAddr, + replace_components: BTreeMap, + }, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InstanceEnsureRequest { + pub properties: InstanceProperties, + pub init: InstanceInitializationMethod, +} + +impl From + for InstanceInitializationMethod +{ + fn from(old: v7::api::InstanceInitializationMethod) -> Self { + match old { + v7::api::InstanceInitializationMethod::Spec { spec } => { + Self::Spec { spec: spec.into() } + } + v7::api::InstanceInitializationMethod::MigrationTarget { + migration_id, + src_addr, + replace_components, + } => Self::MigrationTarget { + migration_id, + src_addr, + replace_components, + }, + } + } +} + +impl From for InstanceEnsureRequest { + fn from(old: v7::api::InstanceEnsureRequest) -> Self { + let v7::api::InstanceEnsureRequest { properties, init } = old; + Self { properties, init: init.into() } + } +} diff --git a/crates/propolis-api-types-versions/src/ext_msi_dest_id/components/board.rs b/crates/propolis-api-types-versions/src/ext_msi_dest_id/components/board.rs new file mode 100644 index 000000000..d5cc861d7 --- /dev/null +++ b/crates/propolis-api-types-versions/src/ext_msi_dest_id/components/board.rs @@ -0,0 +1,307 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! VM mainboard components for the `EXT_MSI_DEST_ID` API version. +//! +//! This version allows the bhyve guest hypervisor interface to carry a set of +//! optional features, and adds a new Hyper-V feature flag. + +use std::collections::BTreeSet; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v1; +use crate::v1::components::board::{Chipset, Cpuid}; + +/// Flags that enable optional bhyve enlightenments. +#[derive( + Clone, + Copy, + Deserialize, + Serialize, + Debug, + JsonSchema, + Ord, + PartialOrd, + Eq, + PartialEq, +)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum BhyveFeatureFlag { + /// Advertise support for 15-bit Extended Destination IDs in MSIs and I/O + /// APIC Redirection Table Entries, allowing guests to address more than + /// 255 vCPUs without interrupt remapping. + ExtMsiDestId, +} + +/// Flags that enable "simple" Hyper-V enlightenments that require no +/// feature-specific configuration. +// +// NOTE: This enum's variants should never have any associated data (note that +// the type doesn't use serde's `tag` and `content` attributes). If a future +// enlightenment requires associated data, it should be put into a +// `HyperVExtendedFeatures` struct (or similar), and the `HyperV` variant of +// `GuestHypervisorInterface` should be extended to `Option`ally include that +// struct. +#[derive( + Clone, + Copy, + Deserialize, + Serialize, + Debug, + JsonSchema, + Ord, + PartialOrd, + Eq, + PartialEq, +)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum HyperVFeatureFlag { + /// Expose the partition reference time MSR and the reference TSC page. + ReferenceTsc, + + /// Advertise support for 15-bit Extended Destination IDs in MSIs and I/O + /// APIC Redirection Table Entries, allowing guests to address more than + /// 255 vCPUs without interrupt remapping. + ExtendedIoapicRte, +} + +/// A hypervisor interface to expose to the guest. +#[derive(Clone, Deserialize, Serialize, Debug, JsonSchema, PartialEq, Eq)] +#[serde( + deny_unknown_fields, + rename_all = "snake_case", + tag = "type", + content = "value" +)] +pub enum GuestHypervisorInterface { + /// Expose a bhyve-like interface ("bhyve bhyve " as the hypervisor ID in + /// leaf 0x4000_0000) with the supplied features enabled. + Bhyve { features: BTreeSet }, + + /// Expose a Hyper-V-compatible hypervisor interface with the supplied + /// features enabled. + HyperV { features: BTreeSet }, +} + +impl Default for GuestHypervisorInterface { + fn default() -> Self { + Self::Bhyve { features: BTreeSet::new() } + } +} + +impl GuestHypervisorInterface { + pub(crate) fn is_default(&self) -> bool { + matches!(self, Self::Bhyve { features } if features.is_empty()) + } +} + +/// A VM's mainboard. +#[derive(Clone, Deserialize, Serialize, Debug, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Board { + /// The number of virtual logical processors attached to this VM. + pub cpus: u8, + + /// The amount of guest RAM attached to this VM. + pub memory_mb: u64, + + /// The chipset to expose to guest software. + pub chipset: Chipset, + + /// The hypervisor platform to expose to the guest. The default is a + /// bhyve-compatible interface with no additional features. + #[serde( + default, + skip_serializing_if = "GuestHypervisorInterface::is_default" + )] + pub guest_hv_interface: GuestHypervisorInterface, + + /// The CPUID values to expose to the guest. If `None`, bhyve will derive + /// default values from the host's CPUID values. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpuid: Option, + // TODO: Processor and NUMA topology. +} + +/// The error returned when a guest hypervisor interface uses features that +/// cannot be expressed in the v1 API types. +#[derive(thiserror::Error, Debug)] +#[error("cannot convert guest hypervisor interface to v1: {reason}")] +pub struct InvalidV1GuestHypervisorInterface { + pub(crate) reason: &'static str, +} + +impl From for HyperVFeatureFlag { + fn from(old: v1::components::board::HyperVFeatureFlag) -> Self { + match old { + v1::components::board::HyperVFeatureFlag::ReferenceTsc => { + Self::ReferenceTsc + } + } + } +} + +impl TryFrom for v1::components::board::HyperVFeatureFlag { + type Error = InvalidV1GuestHypervisorInterface; + + fn try_from(new: HyperVFeatureFlag) -> Result { + match new { + HyperVFeatureFlag::ReferenceTsc => Ok(Self::ReferenceTsc), + HyperVFeatureFlag::ExtendedIoapicRte => { + Err(InvalidV1GuestHypervisorInterface { + reason: "Hyper-V feature extended_ioapic_rte", + }) + } + } + } +} + +impl From + for GuestHypervisorInterface +{ + fn from(old: v1::components::board::GuestHypervisorInterface) -> Self { + match old { + v1::components::board::GuestHypervisorInterface::Bhyve => { + Self::Bhyve { features: BTreeSet::new() } + } + v1::components::board::GuestHypervisorInterface::HyperV { + features, + } => Self::HyperV { + features: features.into_iter().map(Into::into).collect(), + }, + } + } +} + +impl TryFrom + for v1::components::board::GuestHypervisorInterface +{ + type Error = InvalidV1GuestHypervisorInterface; + + fn try_from(new: GuestHypervisorInterface) -> Result { + match new { + GuestHypervisorInterface::Bhyve { features } => { + if !features.is_empty() { + return Err(InvalidV1GuestHypervisorInterface { + reason: "bhyve interface with features", + }); + } + Ok(Self::Bhyve) + } + GuestHypervisorInterface::HyperV { features } => Ok(Self::HyperV { + features: features + .into_iter() + .map(TryInto::try_into) + .collect::>()?, + }), + } + } +} + +impl From for Board { + fn from(old: v1::components::board::Board) -> Self { + let v1::components::board::Board { + cpus, + memory_mb, + chipset, + guest_hv_interface, + cpuid, + } = old; + Self { + cpus, + memory_mb, + chipset, + guest_hv_interface: guest_hv_interface.into(), + cpuid, + } + } +} + +impl TryFrom for v1::components::board::Board { + type Error = InvalidV1GuestHypervisorInterface; + + fn try_from(new: Board) -> Result { + let Board { cpus, memory_mb, chipset, guest_hv_interface, cpuid } = new; + Ok(Self { + cpus, + memory_mb, + chipset, + guest_hv_interface: guest_hv_interface.try_into()?, + cpuid, + }) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn default_interface_is_not_serialized() { + let board = Board { + cpus: 1, + memory_mb: 1024, + chipset: Chipset::default(), + guest_hv_interface: GuestHypervisorInterface::default(), + cpuid: None, + }; + let json = serde_json::to_value(&board).unwrap(); + assert!(json.get("guest_hv_interface").is_none()); + + let roundtrip: Board = serde_json::from_value(json).unwrap(); + assert_eq!(roundtrip.guest_hv_interface, board.guest_hv_interface); + } + + #[test] + fn bhyve_features_roundtrip() { + let iface = GuestHypervisorInterface::Bhyve { + features: [BhyveFeatureFlag::ExtMsiDestId].into(), + }; + let json = serde_json::to_value(&iface).unwrap(); + assert_eq!( + json, + serde_json::json!({ + "type": "bhyve", + "value": { "features": ["ext_msi_dest_id"] } + }) + ); + let roundtrip: GuestHypervisorInterface = + serde_json::from_value(json).unwrap(); + assert_eq!(roundtrip, iface); + } + + #[test] + fn v1_downgrade_rejects_new_features() { + let bhyve = GuestHypervisorInterface::Bhyve { + features: [BhyveFeatureFlag::ExtMsiDestId].into(), + }; + assert!(v1::components::board::GuestHypervisorInterface::try_from( + bhyve + ) + .is_err()); + + let hyperv = GuestHypervisorInterface::HyperV { + features: [ + HyperVFeatureFlag::ReferenceTsc, + HyperVFeatureFlag::ExtendedIoapicRte, + ] + .into(), + }; + assert!(v1::components::board::GuestHypervisorInterface::try_from( + hyperv + ) + .is_err()); + + let hyperv_tsc = GuestHypervisorInterface::HyperV { + features: [HyperVFeatureFlag::ReferenceTsc].into(), + }; + let v1 = v1::components::board::GuestHypervisorInterface::try_from( + hyperv_tsc.clone(), + ) + .unwrap(); + assert_eq!(GuestHypervisorInterface::from(v1), hyperv_tsc); + } +} diff --git a/crates/propolis-api-types-versions/src/ext_msi_dest_id/components/mod.rs b/crates/propolis-api-types-versions/src/ext_msi_dest_id/components/mod.rs new file mode 100644 index 000000000..ee9226076 --- /dev/null +++ b/crates/propolis-api-types-versions/src/ext_msi_dest_id/components/mod.rs @@ -0,0 +1,5 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +pub mod board; diff --git a/crates/propolis-api-types-versions/src/ext_msi_dest_id/instance_spec.rs b/crates/propolis-api-types-versions/src/ext_msi_dest_id/instance_spec.rs new file mode 100644 index 000000000..29ac53787 --- /dev/null +++ b/crates/propolis-api-types-versions/src/ext_msi_dest_id/instance_spec.rs @@ -0,0 +1,80 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::components::board::{Board, InvalidV1GuestHypervisorInterface}; +use crate::v1::instance::InstanceProperties; +use crate::v1::instance_spec::SpecKey; +use crate::v2::instance_spec::SmbiosType1Input; +use crate::v6; +use crate::v6::instance_spec::Component; +use crate::v7; +use crate::v7::instance::InstanceState; + +#[derive(Clone, Deserialize, Serialize, Debug, JsonSchema)] +pub struct InstanceSpec { + pub board: Board, + pub components: BTreeMap, + pub smbios: Option, +} + +#[derive(Clone, Deserialize, Serialize, JsonSchema)] +#[serde(tag = "type", content = "value", rename_all = "snake_case")] +pub enum InstanceSpecStatus { + WaitingForMigrationSource, + Present(InstanceSpec), +} + +#[derive(Clone, Deserialize, Serialize, JsonSchema)] +pub struct InstanceSpecGetResponse { + pub properties: InstanceProperties, + pub state: InstanceState, + pub spec: InstanceSpecStatus, +} + +impl From for InstanceSpec { + fn from(old: v6::instance_spec::InstanceSpec) -> Self { + let v6::instance_spec::InstanceSpec { board, components, smbios } = old; + Self { board: board.into(), components, smbios } + } +} + +impl TryFrom for v6::instance_spec::InstanceSpec { + type Error = InvalidV1GuestHypervisorInterface; + + fn try_from(new: InstanceSpec) -> Result { + let InstanceSpec { board, components, smbios } = new; + Ok(Self { board: board.try_into()?, components, smbios }) + } +} + +impl TryFrom for v7::instance_spec::InstanceSpecStatus { + type Error = InvalidV1GuestHypervisorInterface; + + fn try_from(new: InstanceSpecStatus) -> Result { + Ok(match new { + InstanceSpecStatus::WaitingForMigrationSource => { + Self::WaitingForMigrationSource + } + InstanceSpecStatus::Present(spec) => { + Self::Present(spec.try_into()?) + } + }) + } +} + +impl TryFrom + for v7::instance_spec::InstanceSpecGetResponse +{ + type Error = InvalidV1GuestHypervisorInterface; + + fn try_from(new: InstanceSpecGetResponse) -> Result { + let InstanceSpecGetResponse { properties, state, spec } = new; + Ok(Self { properties, state, spec: spec.try_into()? }) + } +} diff --git a/crates/propolis-api-types-versions/src/ext_msi_dest_id/mod.rs b/crates/propolis-api-types-versions/src/ext_msi_dest_id/mod.rs new file mode 100644 index 000000000..427962903 --- /dev/null +++ b/crates/propolis-api-types-versions/src/ext_msi_dest_id/mod.rs @@ -0,0 +1,13 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Version `EXT_MSI_DEST_ID` of the Propolis Server API. +//! +//! This version allows the bhyve guest hypervisor interface to specify a set +//! of optional features (initially, 15-bit Extended MSI Destination IDs), and +//! adds the `extended_ioapic_rte` Hyper-V feature flag. + +pub mod api; +pub mod components; +pub mod instance_spec; diff --git a/crates/propolis-api-types-versions/src/latest.rs b/crates/propolis-api-types-versions/src/latest.rs index 04de08a68..c6fcb71eb 100644 --- a/crates/propolis-api-types-versions/src/latest.rs +++ b/crates/propolis-api-types-versions/src/latest.rs @@ -17,13 +17,15 @@ pub mod components { } pub mod board { - pub use crate::v1::components::board::Board; pub use crate::v1::components::board::Chipset; pub use crate::v1::components::board::Cpuid; pub use crate::v1::components::board::CpuidEntry; - pub use crate::v1::components::board::GuestHypervisorInterface; - pub use crate::v1::components::board::HyperVFeatureFlag; pub use crate::v1::components::board::I440Fx; + + pub use crate::v8::components::board::BhyveFeatureFlag; + pub use crate::v8::components::board::Board; + pub use crate::v8::components::board::GuestHypervisorInterface; + pub use crate::v8::components::board::HyperVFeatureFlag; } pub mod devices { @@ -66,14 +68,15 @@ pub mod instance { pub use crate::v1::instance::InstanceStateChange; pub use crate::v1::instance::InstanceStateMonitorRequest; - pub use crate::v7::api::InstanceEnsureRequest; - pub use crate::v7::api::InstanceInitializationMethod; pub use crate::v7::instance::Instance; pub use crate::v7::instance::InstanceGetResponse; pub use crate::v7::instance::InstanceState; pub use crate::v7::instance::InstanceStateMonitorResponse; pub use crate::v7::instance::InstanceStateRequested; pub use crate::v7::instance::ReplacementComponent; + + pub use crate::v8::api::InstanceEnsureRequest; + pub use crate::v8::api::InstanceInitializationMethod; } pub mod instance_spec { @@ -87,10 +90,10 @@ pub mod instance_spec { pub use crate::v2::instance_spec::SmbiosType1Input; pub use crate::v6::instance_spec::Component; - pub use crate::v6::instance_spec::InstanceSpec; - pub use crate::v7::instance_spec::InstanceSpecGetResponse; - pub use crate::v7::instance_spec::InstanceSpecStatus; + pub use crate::v8::instance_spec::InstanceSpec; + pub use crate::v8::instance_spec::InstanceSpecGetResponse; + pub use crate::v8::instance_spec::InstanceSpecStatus; } pub mod migration { diff --git a/crates/propolis-api-types-versions/src/lib.rs b/crates/propolis-api-types-versions/src/lib.rs index 55bc4f31f..c71f26b85 100644 --- a/crates/propolis-api-types-versions/src/lib.rs +++ b/crates/propolis-api-types-versions/src/lib.rs @@ -43,3 +43,5 @@ pub mod v5; pub mod v6; #[path = "openapi_lint/mod.rs"] pub mod v7; +#[path = "ext_msi_dest_id/mod.rs"] +pub mod v8; diff --git a/crates/propolis-config-toml/src/spec.rs b/crates/propolis-config-toml/src/spec.rs index 9a254c8b0..c7d78257c 100644 --- a/crates/propolis-config-toml/src/spec.rs +++ b/crates/propolis-config-toml/src/spec.rs @@ -12,11 +12,12 @@ use std::{ use crate::HypervisorInterface; use propolis_client::{ instance_spec::{ - BootOrderEntry, BootSettings, Component, Cpuid, CpuidVendor, - DlpiNetworkBackend, FileStorageBackend, GuestHypervisorInterface, - HyperVFeatureFlag, MigrationFailureInjector, NvmeDisk, P9fs, PciPath, - PciPciBridge, SoftNpuP9, SoftNpuPciPort, SoftNpuPort, SpecKey, - VirtioDisk, VirtioNetworkBackend, VirtioNic, VirtioSocket, + BhyveFeatureFlag, BootOrderEntry, BootSettings, Component, Cpuid, + CpuidVendor, DlpiNetworkBackend, FileStorageBackend, + GuestHypervisorInterface, HyperVFeatureFlag, MigrationFailureInjector, + NvmeDisk, P9fs, PciPath, PciPciBridge, SoftNpuP9, SoftNpuPciPort, + SoftNpuPort, SpecKey, VirtioDisk, VirtioNetworkBackend, VirtioNic, + VirtioSocket, }, support::nvme_serial_from_str, }; @@ -115,20 +116,24 @@ impl TryFrom<&super::Config> for SpecConfig { .hv_interface .as_ref() .map(|hv| match hv { - HypervisorInterface::Bhyve { ext_msi_dest_id: _ } => { - // XXX(luqman) update api spec version to accept features - GuestHypervisorInterface::Bhyve + HypervisorInterface::Bhyve { ext_msi_dest_id } => { + let mut features = BTreeSet::new(); + if *ext_msi_dest_id { + features.insert(BhyveFeatureFlag::ExtMsiDestId); + } + GuestHypervisorInterface::Bhyve { features } } HypervisorInterface::HyperV { reference_tsc, - extended_ioapic_rte: _, + extended_ioapic_rte, } => { - // XXX(luqman) update api spec version to accept - // extended_ioapic_rte let mut features = BTreeSet::new(); if *reference_tsc { features.insert(HyperVFeatureFlag::ReferenceTsc); } + if *extended_ioapic_rte { + features.insert(HyperVFeatureFlag::ExtendedIoapicRte); + } GuestHypervisorInterface::HyperV { features } } }) diff --git a/crates/propolis-server-api/src/lib.rs b/crates/propolis-server-api/src/lib.rs index 487a67339..8813c681c 100644 --- a/crates/propolis-server-api/src/lib.rs +++ b/crates/propolis-server-api/src/lib.rs @@ -8,7 +8,7 @@ use dropshot::{ WebsocketChannelResult, WebsocketConnection, }; use dropshot_api_manager_types::api_versions; -use propolis_api_types_versions::{latest, v1, v2, v3, v6, v7}; +use propolis_api_types_versions::{latest, v1, v2, v3, v6, v7, v8}; api_versions!([ // WHEN CHANGING THE API (part 1 of 2): @@ -22,6 +22,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (8, EXT_MSI_DEST_ID), (7, OPENAPI_LINT), (6, NVME_WRITE_CACHE), (5, CRUCIBLE_VOLUME_INFO), @@ -50,7 +51,7 @@ pub trait PropolisServerApi { #[endpoint { method = PUT, path = "/instance", - versions = VERSION_OPENAPI_LINT.. + versions = VERSION_EXT_MSI_DEST_ID.. }] async fn instance_ensure( rqctx: RequestContext, @@ -60,6 +61,26 @@ pub trait PropolisServerApi { HttpError, >; + #[endpoint { + operation_id = "instance_ensure", + method = PUT, + path = "/instance", + versions = VERSION_OPENAPI_LINT..VERSION_EXT_MSI_DEST_ID + }] + async fn instance_ensure_v7( + rqctx: RequestContext, + request: TypedBody, + ) -> Result< + HttpResponseCreated, + HttpError, + > { + Self::instance_ensure( + rqctx, + request.map(v8::api::InstanceEnsureRequest::from), + ) + .await + } + #[endpoint { operation_id = "instance_ensure", method = PUT, @@ -73,7 +94,7 @@ pub trait PropolisServerApi { HttpResponseCreated, HttpError, > { - Self::instance_ensure( + Self::instance_ensure_v7( rqctx, request.map(v7::api::InstanceEnsureRequest::from), ) @@ -143,7 +164,7 @@ pub trait PropolisServerApi { #[endpoint { method = GET, path = "/instance/spec", - versions = VERSION_OPENAPI_LINT.. + versions = VERSION_EXT_MSI_DEST_ID.. }] async fn instance_spec_get( rqctx: RequestContext, @@ -152,6 +173,35 @@ pub trait PropolisServerApi { HttpError, >; + #[endpoint { + operation_id = "instance_spec_get", + method = GET, + path = "/instance/spec", + versions = VERSION_OPENAPI_LINT..VERSION_EXT_MSI_DEST_ID + }] + async fn instance_spec_get_v7( + rqctx: RequestContext, + ) -> Result< + HttpResponseOk, + HttpError, + > { + let v8_response = Self::instance_spec_get(rqctx).await?.0; + let v7_response: v7::instance_spec::InstanceSpecGetResponse = + v8_response.try_into().map_err( + |e: v8::components::board::InvalidV1GuestHypervisorInterface| { + HttpError::for_client_error( + None, + ClientErrorStatusCode::BAD_REQUEST, + format!( + "instance spec cannot be expressed to v7 \ + clients: {e}" + ), + ) + }, + )?; + Ok(HttpResponseOk(v7_response)) + } + #[endpoint { operation_id = "instance_spec_get", method = GET, @@ -164,7 +214,7 @@ pub trait PropolisServerApi { HttpResponseOk, HttpError, > { - Ok(Self::instance_spec_get(rqctx) + Ok(Self::instance_spec_get_v7(rqctx) .await? .map(v6::instance_spec::InstanceSpecGetResponse::from)) } diff --git a/openapi/propolis-server/propolis-server-7.0.0-712b68.json.gitstub b/openapi/propolis-server/propolis-server-7.0.0-712b68.json.gitstub new file mode 100644 index 000000000..ad6ba570e --- /dev/null +++ b/openapi/propolis-server/propolis-server-7.0.0-712b68.json.gitstub @@ -0,0 +1 @@ +24caf8a9574b055e02f4619af03aacc3d19d9999:openapi/propolis-server/propolis-server-7.0.0-712b68.json diff --git a/openapi/propolis-server/propolis-server-7.0.0-712b68.json b/openapi/propolis-server/propolis-server-8.0.0-afac14.json similarity index 97% rename from openapi/propolis-server/propolis-server-7.0.0-712b68.json rename to openapi/propolis-server/propolis-server-8.0.0-afac14.json index 8b222f1bb..3c6a7e0d7 100644 --- a/openapi/propolis-server/propolis-server-7.0.0-712b68.json +++ b/openapi/propolis-server/propolis-server-8.0.0-afac14.json @@ -7,7 +7,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "7.0.0" + "version": "8.0.0" }, "paths": { "/instance": { @@ -432,6 +432,18 @@ }, "components": { "schemas": { + "BhyveFeatureFlag": { + "description": "Flags that enable optional bhyve enlightenments.", + "oneOf": [ + { + "description": "Advertise support for 15-bit Extended Destination IDs in MSIs and I/O APIC Redirection Table Entries, allowing guests to address more than 255 vCPUs without interrupt remapping.", + "type": "string", + "enum": [ + "ext_msi_dest_id" + ] + } + ] + }, "BlobStorageBackend": { "description": "A storage backend for a disk whose initial contents are given explicitly by the specification.", "type": "object", @@ -479,7 +491,7 @@ "minimum": 0 }, "guest_hv_interface": { - "description": "The hypervisor platform to expose to the guest. The default is a bhyve-compatible interface with no additional features.\n\nFor compatibility with older versions of Propolis, this field is only serialized if it specifies a non-default interface.", + "description": "The hypervisor platform to expose to the guest. The default is a bhyve-compatible interface with no additional features.", "allOf": [ { "$ref": "#/components/schemas/GuestHypervisorInterface" @@ -1190,7 +1202,7 @@ "description": "A hypervisor interface to expose to the guest.", "oneOf": [ { - "description": "Expose a bhyve-like interface (\"bhyve bhyve \" as the hypervisor ID in leaf 0x4000_0000 and no additional leaves or features).", + "description": "Expose a bhyve-like interface (\"bhyve bhyve \" as the hypervisor ID in leaf 0x4000_0000) with the supplied features enabled.", "type": "object", "properties": { "type": { @@ -1198,10 +1210,27 @@ "enum": [ "bhyve" ] + }, + "value": { + "type": "object", + "properties": { + "features": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BhyveFeatureFlag" + }, + "uniqueItems": true + } + }, + "required": [ + "features" + ], + "additionalProperties": false } }, "required": [ - "type" + "type", + "value" ], "additionalProperties": false }, @@ -1242,9 +1271,21 @@ }, "HyperVFeatureFlag": { "description": "Flags that enable \"simple\" Hyper-V enlightenments that require no feature-specific configuration.", - "type": "string", - "enum": [ - "reference_tsc" + "oneOf": [ + { + "description": "Expose the partition reference time MSR and the reference TSC page.", + "type": "string", + "enum": [ + "reference_tsc" + ] + }, + { + "description": "Advertise support for 15-bit Extended Destination IDs in MSIs and I/O APIC Redirection Table Entries, allowing guests to address more than 255 vCPUs without interrupt remapping.", + "type": "string", + "enum": [ + "extended_ioapic_rte" + ] + } ] }, "I440Fx": { diff --git a/openapi/propolis-server/propolis-server-latest.json b/openapi/propolis-server/propolis-server-latest.json index dfaf8de31..d19204b58 120000 --- a/openapi/propolis-server/propolis-server-latest.json +++ b/openapi/propolis-server/propolis-server-latest.json @@ -1 +1 @@ -propolis-server-7.0.0-712b68.json \ No newline at end of file +propolis-server-8.0.0-afac14.json \ No newline at end of file From dcba5c8d58afb70e2a0dc9692a28b4a371c5ccf2 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Wed, 18 Nov 2026 14:39:19 -0800 Subject: [PATCH 6/8] propolis: make Machine::vcpus private --- bin/propolis-server/src/lib/initializer.rs | 2 +- bin/propolis-server/src/lib/vcpu_tasks.rs | 10 ++++----- bin/propolis-server/src/lib/vm/ensure.rs | 3 +-- bin/propolis-server/src/lib/vm/objects.rs | 2 +- bin/propolis-standalone/src/main.rs | 11 +++++---- bin/propolis-standalone/src/snapshot.rs | 2 +- lib/propolis/src/vmm/machine.rs | 26 ++++++++++++++++++---- 7 files changed, 36 insertions(+), 20 deletions(-) diff --git a/bin/propolis-server/src/lib/initializer.rs b/bin/propolis-server/src/lib/initializer.rs index 0da580375..4f28258e1 100644 --- a/bin/propolis-server/src/lib/initializer.rs +++ b/bin/propolis-server/src/lib/initializer.rs @@ -1585,7 +1585,7 @@ impl MachineInitializer<'_> { /// tracking their kstats. pub async fn initialize_cpus(&mut self) -> Result<(), MachineInitError> { let hv_interface = self.machine.guest_hv_interface.as_ref(); - for vcpu in self.machine.vcpus.iter() { + for vcpu in self.machine.vcpus() { // Report that the guest is running on bhyve. // // The CPUID set in the spec is not allowed to contain any leaves in diff --git a/bin/propolis-server/src/lib/vcpu_tasks.rs b/bin/propolis-server/src/lib/vcpu_tasks.rs index 3870033c3..c4261ebd3 100644 --- a/bin/propolis-server/src/lib/vcpu_tasks.rs +++ b/bin/propolis-server/src/lib/vcpu_tasks.rs @@ -50,22 +50,22 @@ impl VcpuTasks { // We take in an `Option>` but a `Vec>` is more // convenient for spawning below, so we have to shuffle values a bit.. - let mut bindings = vec![None; machine.vcpus.len()]; + let mut bindings = vec![None; machine.num_vcpus()]; if let Some(bind_cpus) = bind_cpus { - if bind_cpus.len() != machine.vcpus.len() { + if bind_cpus.len() != machine.num_vcpus() { return Err(VcpuTaskError::CpuBindingMismatch { bindings: bind_cpus.len(), - vcpus: machine.vcpus.len(), + vcpus: machine.num_vcpus(), }); } - for i in 0..machine.vcpus.len() { + for i in 0..machine.num_vcpus() { bindings[i] = Some(bind_cpus[i]); } } let mut tasks = Vec::new(); for (vcpu, bind_cpu) in - machine.vcpus.iter().map(Arc::clone).zip(bindings) + machine.vcpus().into_iter().cloned().zip(bindings) { let (task, ctrl) = propolis::tasks::TaskHdl::new_held(Some(vcpu.barrier_fn())); diff --git a/bin/propolis-server/src/lib/vm/ensure.rs b/bin/propolis-server/src/lib/vm/ensure.rs index 8481d1f37..b3622d82d 100644 --- a/bin/propolis-server/src/lib/vm/ensure.rs +++ b/bin/propolis-server/src/lib/vm/ensure.rs @@ -613,8 +613,7 @@ async fn initialize_vm_objects( let total_cpus = pbind::online_cpus()?; let vcpu_count: i32 = machine - .vcpus - .len() + .num_vcpus() .try_into() .map_err(|_| anyhow::anyhow!("more than 2^31 vCPUs"))?; diff --git a/bin/propolis-server/src/lib/vm/objects.rs b/bin/propolis-server/src/lib/vm/objects.rs index 49ba61c84..21219b61c 100644 --- a/bin/propolis-server/src/lib/vm/objects.rs +++ b/bin/propolis-server/src/lib/vm/objects.rs @@ -400,7 +400,7 @@ impl VmObjectsLocked { /// Resets a VM's kernel vCPU objects to their initial states. fn reset_vcpu_state(&self) { - for vcpu in self.machine.vcpus.iter() { + for vcpu in self.machine.vcpus() { info!(self.log, "resetting vCPU {}", vcpu.id); vcpu.activate().unwrap(); vcpu.reboot_state().unwrap(); diff --git a/bin/propolis-standalone/src/main.rs b/bin/propolis-standalone/src/main.rs index eb699fd29..4048b7629 100644 --- a/bin/propolis-standalone/src/main.rs +++ b/bin/propolis-standalone/src/main.rs @@ -311,8 +311,7 @@ impl Instance { let total_cpus = pbind::online_cpus().expect("can get processor count"); let vcpu_count: i32 = machine - .vcpus - .len() + .num_vcpus() .try_into() .expect("vCPU count <= MAXCPU < i32::MAX"); @@ -329,12 +328,12 @@ impl Instance { bind_cpus } Some(config::BindingStrategy::Any) | None => { - vec![None; machine.vcpus.len()] + vec![None; machine.num_vcpus()] } }; for (vcpu, bind_cpu) in - machine.vcpus.iter().map(Arc::clone).zip(bind_cpus) + machine.vcpus().into_iter().cloned().zip(bind_cpus) { let (task, ctrl) = propolis::tasks::TaskHdl::new_held(Some(vcpu.barrier_fn())); @@ -1218,7 +1217,7 @@ fn setup_instance( drop(romfp); // Add vCPUs to inventory, since they count as devices - for vcpu in machine.vcpus.iter() { + for vcpu in machine.vcpus() { guard.inventory.register_instance(vcpu, &vcpu.id.to_string()) } @@ -1592,7 +1591,7 @@ fn setup_instance( .add_cpuid(&mut base_profile) .context("failed to add hypervisor cpuid leaves")?; - for vcpu in machine.vcpus.iter() { + for vcpu in machine.vcpus() { let vcpu_profile = propolis::cpuid::Specializer::new() .with_vcpu_count( std::num::NonZeroU8::new(config.main.cpus).unwrap(), diff --git a/bin/propolis-standalone/src/snapshot.rs b/bin/propolis-standalone/src/snapshot.rs index a390dc32c..176a5c0f5 100644 --- a/bin/propolis-standalone/src/snapshot.rs +++ b/bin/propolis-standalone/src/snapshot.rs @@ -263,7 +263,7 @@ pub(crate) fn restore( hdl.pause()?; // Ensure vCPUs are in the active state - for vcpu in machine.vcpus.iter() { + for vcpu in machine.vcpus() { vcpu.activate().context("Failed to activate vCPU")?; } diff --git a/lib/propolis/src/vmm/machine.rs b/lib/propolis/src/vmm/machine.rs index 8a7492d1c..36a150c8c 100644 --- a/lib/propolis/src/vmm/machine.rs +++ b/lib/propolis/src/vmm/machine.rs @@ -29,7 +29,8 @@ pub const MAX_PHYSMEM: usize = 0x1000_0000_0000; /// The aggregate representation of a virtual machine. pub struct Machine { pub hdl: Arc, - pub vcpus: Vec>, + // Private so the invariants documented on `vcpus()` can't be broken. + vcpus: Vec>, pub map_physmem: PhysMap, pub bus_mmio: Arc, @@ -50,6 +51,23 @@ impl Machine { Ok(()) } + /// The machine's vCPUs, sorted by ascending ID. There is always at least + /// one, and the first is the bootstrap processor (vCPU 0). + pub fn vcpus(&self) -> impl IntoIterator> + '_ { + &self.vcpus + } + + /// The bootstrap processor (vCPU 0). + pub fn bsp(&self) -> &Arc { + // When the Machine is created, we're guaranteed at least one vcpu + &self.vcpus[0] + } + + /// The number of vCPUs in the machine. + pub fn num_vcpus(&self) -> usize { + self.vcpus.len() + } + /// (Re)Initialize vCPUs per x86 spec pub fn vcpu_x86_setup(&self) -> Result<()> { for vcpu in self.vcpus.iter() { @@ -95,10 +113,10 @@ impl Machine { } } + /// Send a Non Maskable Interrupt (NMI) to the BSP (vCPU 0). pub fn inject_nmi(&self) -> Result<()> { - // When the Machine is created, we're guaranteed at least one vcpu, so - // just send the NMI to the first one. - self.vcpus[0].inject_nmi() + // The BSP (vCPU 0) always exists so just the NMI to it. + self.bsp().inject_nmi() } } impl Drop for Machine { From cb5352c0ae00f476043dc28663848971169136b0 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Wed, 18 Nov 2026 15:46:53 -0800 Subject: [PATCH 7/8] propolis-standalone: allow non-contiguous vCPU IDs --- bin/propolis-server/src/lib/initializer.rs | 8 +- bin/propolis-standalone/README.md | 4 + bin/propolis-standalone/src/config.rs | 8 ++ bin/propolis-standalone/src/main.rs | 36 ++++-- crates/bhyve-api/sys/src/structs.rs | 9 ++ lib/propolis/src/cpuid.rs | 134 ++++++++++++++++++++- lib/propolis/src/firmware/acpi/madt.rs | 21 +++- lib/propolis/src/hw/qemu/fwcfg.rs | 5 +- lib/propolis/src/vmm/hdl.rs | 11 ++ lib/propolis/src/vmm/machine.rs | 78 +++++++++++- 10 files changed, 285 insertions(+), 29 deletions(-) diff --git a/bin/propolis-server/src/lib/initializer.rs b/bin/propolis-server/src/lib/initializer.rs index 4f28258e1..f66b814ff 100644 --- a/bin/propolis-server/src/lib/initializer.rs +++ b/bin/propolis-server/src/lib/initializer.rs @@ -1469,7 +1469,6 @@ impl MachineInitializer<'_> { fn generate_acpi_tables( &self, acpi_variant: acpi::AcpiVariant, - cpus: u8, ) -> Result { let (lowmem, _) = get_spec_guest_ram_limits(self.spec); let generators: Vec<_> = self @@ -1491,9 +1490,10 @@ impl MachineInitializer<'_> { PCI_MMIO32_END as u64, )?; + let apic_ids = self.machine.apic_ids(); let config = &fwcfg::formats::AcpiConfig { acpi_variant, - num_cpus: cpus, + apic_ids: &apic_ids, pci_window_32, pci_window_64: fwcfg::formats::PciWindow::empty(), dsdt_generators: &generators, @@ -1550,7 +1550,7 @@ impl MachineInitializer<'_> { .insert_named("etc/e820", e820_entry) .map_err(|e| MachineInitError::FwcfgInsertFailed("e820", e))?; - let acpi_entries = self.generate_acpi_tables(acpi_variant, cpus)?; + let acpi_entries = self.generate_acpi_tables(acpi_variant)?; fwcfg.insert_named("etc/acpi/tables", acpi_entries.tables).map_err( |e| MachineInitError::FwcfgInsertFailed("acpi/tables", e), )?; @@ -1585,6 +1585,7 @@ impl MachineInitializer<'_> { /// tracking their kstats. pub async fn initialize_cpus(&mut self) -> Result<(), MachineInitError> { let hv_interface = self.machine.guest_hv_interface.as_ref(); + let max_vcpuid = self.machine.max_vcpu_id(); for vcpu in self.machine.vcpus() { // Report that the guest is running on bhyve. // @@ -1615,6 +1616,7 @@ impl MachineInitializer<'_> { NonZeroU8::new(self.spec.board.cpus).unwrap(), true, ) + .with_max_vcpuid(max_vcpuid) .with_vcpuid(vcpu.id) .with_cache_topo() .clear_cpu_topo(TopoKind::iter()) diff --git a/bin/propolis-standalone/README.md b/bin/propolis-standalone/README.md index 4990d7885..2e3241dd4 100644 --- a/bin/propolis-standalone/README.md +++ b/bin/propolis-standalone/README.md @@ -26,6 +26,10 @@ memory = 1024 # Override boot order (via communication to OVMF bootrom) # boot_order = ["net0", "block0"] +# Assign specific (possibly sparse) IDs to the vCPUs, which bhyve also uses as +# their APIC IDs. There must be `cpus` entries, including 0 (default: 0..cpus) +# vcpu_ids = [0, 1, 128, 129] + [block_dev.alpine_iso] type = "file" path = "/path/to/alpine-extended-3.12.0-x86_64.iso" diff --git a/bin/propolis-standalone/src/config.rs b/bin/propolis-standalone/src/config.rs index 9c1182427..e65422f3a 100644 --- a/bin/propolis-standalone/src/config.rs +++ b/bin/propolis-standalone/src/config.rs @@ -51,6 +51,14 @@ impl Config { pub struct Main { pub name: String, pub cpus: u8, + /// IDs to assign to the VM's vCPUs. There must be `cpus` of them and they + /// must include 0. bhyve uses a vCPU's ID as its APIC ID, so sparse IDs + /// (e.g. `[0, 1, 128, 129]`) can be used to test guest handling of + /// non-contiguous APIC IDs. IDs must currently be below 255, since higher + /// APIC IDs require x2APIC MADT entries, which aren't yet supported. + /// + /// Default: `0..cpus` + pub vcpu_ids: Option>, pub bootrom: String, pub bootrom_version: Option, pub memory: usize, diff --git a/bin/propolis-standalone/src/main.rs b/bin/propolis-standalone/src/main.rs index 4048b7629..0ee651155 100644 --- a/bin/propolis-standalone/src/main.rs +++ b/bin/propolis-standalone/src/main.rs @@ -777,7 +777,7 @@ impl Instance { fn build_machine( log: &slog::Logger, name: &str, - max_cpu: u8, + vcpu_ids: &[u16], lowmem: usize, highmem: usize, use_reservoir: bool, @@ -791,7 +791,7 @@ fn build_machine( ..Default::default() }, )? - .max_cpus(max_cpu)? + .vcpu_ids(vcpu_ids.iter().copied())? .add_mem_region(0, lowmem, "lowmem")? .add_rom_region(0x1_0000_0000 - MAX_ROM_SIZE, MAX_ROM_SIZE, "bootrom")? .add_mmio_region(0xc000_0000, 0x2000_0000, "dev32")? @@ -1110,7 +1110,7 @@ fn generate_bootorder( fn generate_acpi_tables( acpi_variant: acpi::AcpiVariant, - num_cpus: u8, + apic_ids: &[u32], lowmem: usize, inventory: &Inventory, device_metadata: &DeviceMetadataMap, @@ -1138,7 +1138,7 @@ fn generate_acpi_tables( let config = &fwcfg::formats::AcpiConfig { acpi_variant, - num_cpus, + apic_ids, pci_window_32, pci_window_64: fwcfg::formats::PciWindow::empty(), dsdt_generators: &generators, @@ -1156,6 +1156,26 @@ fn setup_instance( ) -> anyhow::Result<(Instance, Arc)> { let vm_name = &config.main.name; let cpus = config.main.cpus; + let vcpu_ids: Vec = match &config.main.vcpu_ids { + Some(ids) => { + if ids.len() != usize::from(cpus) { + anyhow::bail!( + "vcpu_ids has {} entries but cpus is {cpus}", + ids.len() + ); + } + // bhyve uses vCPU IDs as APIC IDs, and the MADT can't yet + // describe APIC IDs that require x2APIC (propolis#956). + if let Some(id) = ids.iter().find(|&&id| id >= 0xff) { + anyhow::bail!( + "vcpu_ids contains {id}, but IDs >= 255 are not yet \ + supported" + ); + } + ids.clone() + } + None => (0..u16::from(cpus)).collect(), + }; let memsize: usize = config.main.memory * MB; let lowmem = memsize.min(3 * GB); @@ -1180,11 +1200,11 @@ fn setup_instance( } slog::info!(log, "Creating VM with {} vCPUs, {} lowmem, {} highmem", - cpus, lowmem, highmem;); + cpus, lowmem, highmem; "vcpu_ids" => ?vcpu_ids); let machine = build_machine( log, vm_name, - cpus, + &vcpu_ids, lowmem, highmem, use_reservoir, @@ -1551,7 +1571,7 @@ fn setup_instance( let acpi_entries = generate_acpi_tables( config.main.acpi_variant, - cpus, + &machine.apic_ids(), lowmem, &guard.inventory, &device_metadata, @@ -1591,12 +1611,14 @@ fn setup_instance( .add_cpuid(&mut base_profile) .context("failed to add hypervisor cpuid leaves")?; + let max_vcpuid = machine.max_vcpu_id(); for vcpu in machine.vcpus() { let vcpu_profile = propolis::cpuid::Specializer::new() .with_vcpu_count( std::num::NonZeroU8::new(config.main.cpus).unwrap(), true, ) + .with_max_vcpuid(max_vcpuid) .with_vcpuid(vcpu.id) .with_cache_topo() .clear_cpu_topo(cpuid::TopoKind::iter()) diff --git a/crates/bhyve-api/sys/src/structs.rs b/crates/bhyve-api/sys/src/structs.rs index 86efee0c3..c6f5a4198 100644 --- a/crates/bhyve-api/sys/src/structs.rs +++ b/crates/bhyve-api/sys/src/structs.rs @@ -304,6 +304,15 @@ pub struct vm_reinit { pub flags: u64, } +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub struct vm_cpu_topology { + pub sockets: u16, + pub cores: u16, + pub threads: u16, + pub maxcpus: u16, +} + #[repr(C)] #[derive(Copy, Clone, Default)] pub struct vm_vcpu_reset { diff --git a/lib/propolis/src/cpuid.rs b/lib/propolis/src/cpuid.rs index 7ec98276e..897c31fc8 100644 --- a/lib/propolis/src/cpuid.rs +++ b/lib/propolis/src/cpuid.rs @@ -57,6 +57,7 @@ pub enum SpecializeError { pub struct Specializer { has_smt: bool, num_vcpu: Option, + max_vcpuid: Option, vcpuid: Option, cpu_topo_populate: BTreeSet, cpu_topo_clear: BTreeSet, @@ -72,12 +73,38 @@ impl Specializer { Self { num_vcpu: Some(count), has_smt, ..self } } + /// Specify the largest vCPU ID in the instance. + /// + /// bhyve uses a vCPU's ID as its APIC ID. If the instance's vCPU IDs are + /// not contiguous from 0, the topology information must cover the whole + /// range of APIC IDs rather than just the number of vCPUs, so that the + /// guest sees all of its vCPUs in one package sharing one L3 cache. + /// + /// If this is not specified, vCPU IDs are assumed to be contiguous. + pub fn with_max_vcpuid(self, max_vcpuid: i32) -> Self { + let max_vcpuid = + u32::try_from(max_vcpuid).expect("vCPU IDs are non-negative"); + Self { max_vcpuid: Some(max_vcpuid), ..self } + } + /// Specify vCPU ID to specialize for pub fn with_vcpuid(self, vcpuid: i32) -> Self { - assert!((vcpuid as usize) < crate::vcpu::MAXCPU); + // bhyve vCPU IDs may be sparse, so this can't be checked against the + // vCPU count. + assert!(vcpuid >= 0); Self { vcpuid: Some(vcpuid), ..self } } + /// The number of APIC IDs spanned by the instance's vCPUs: one more than + /// the largest vCPU ID, or the number of vCPUs if IDs are contiguous. + fn apic_id_span(&self) -> Option { + let num_vcpu = u32::from(self.num_vcpu?.get()); + Some(match self.max_vcpuid { + Some(max) => num_vcpu.max(max + 1), + None => num_vcpu, + }) + } + /// Specify CPU topology types to render into the specialized [CpuidSet] /// /// Without basic information such as the number of vCPUs (set by @@ -163,7 +190,7 @@ impl Specializer { set: &mut CpuidSet, ) -> Result<(), SpecializeError> { assert!(self.do_cache_topo); - let num = self.num_vcpu.unwrap().get(); + let span = self.apic_id_span().unwrap(); for ecx in 0..u32::MAX { match set.get_mut(CpuidIdent::subleaf(0x8000001d, ecx)) { None => break, @@ -179,9 +206,11 @@ impl Specializer { } } 0b011 => { - // L3 shared by all vCPUs + // L3 shared by all vCPUs. Guests find the vCPUs + // that share a cache by masking off low bits of + // their APIC IDs, so this must span all of them. // TODO: segregate by sockets, if configured - u32::from(num) + span } _ => { // unceremonious handling of unexpected cache levels @@ -205,6 +234,11 @@ impl Specializer { .ok_or(SpecializeError::MissingVcpuCount) .map(|n| u32::from(n.get()))?; + // The number of APIC IDs spanned by the guest's vCPUs. This is + // `num_vcpu` unless the vCPU IDs are sparse. + let apic_id_span = + self.apic_id_span().expect("num_vcpu was checked above"); + // The number of logical processors the guest should see. If we // indicate that SMT is enabled, then vCPUs are presented as pairs // of sibling threads on vproc-many processors. @@ -351,7 +385,10 @@ impl Specializer { } } else { subleaf.eax &= !LEAF4_EAX_VCPU_MASK; - let shifted_vcpu = (num_vcpu - 1) << 14; + // Guests find the vCPUs sharing this cache by + // masking off low bits of their APIC IDs, so this + // must span all of them. + let shifted_vcpu = (apic_id_span - 1) << 14; if shifted_vcpu & !LEAF4_EAX_VCPU_MASK != 0 { return Err( SpecializeError::IncompatibleTopology { @@ -415,6 +452,13 @@ impl Specializer { // you should trip over this quickly if you've bumped up // VM_MAXCPU and you may have hardware to compare against at // that point. + // + // If the vCPU IDs (and so APIC IDs) are sparse, though, + // widen the shift to cover the largest one. Otherwise + // vCPUs with APIC IDs of 256 and above would appear to be + // in a different package. + let core_shift = 8u32 + .max(u32::BITS - (apic_id_span - 1).leading_zeros()); if num_vcpu >= 256 { return Err(SpecializeError::IncompatibleTopology { leaf, @@ -428,7 +472,7 @@ impl Specializer { set.insert( CpuidIdent::subleaf(leaf, 1), CpuidValues { - eax: 0x8, + eax: core_shift, ebx: num_vcpu, ecx: 0x201, edx: vcpuid, @@ -513,3 +557,81 @@ pub fn parse_brand_string( // trim any bounding whitespace which remains Ok(untrimmed.trim().to_string()) } + +#[cfg(test)] +mod test { + use super::*; + + /// L1 data cache and unified L3 cache descriptors for leaf 4. + const LEAF4_L1D_EAX: u32 = 0x21; + const LEAF4_L3_EAX: u32 = 0x63; + + fn base_set() -> CpuidSet { + let mut set = CpuidSet::new(CpuidVendor::Intel); + set.insert(CpuidIdent::leaf(1), CpuidValues::default()).unwrap(); + for (i, eax) in [LEAF4_L1D_EAX, LEAF4_L3_EAX].into_iter().enumerate() { + set.insert( + CpuidIdent::subleaf(4, i as u32), + CpuidValues { eax, ..Default::default() }, + ) + .unwrap(); + } + set.insert(CpuidIdent::subleaf(4, 2), CpuidValues::default()).unwrap(); + set.insert(CpuidIdent::subleaf(0xb, 0), CpuidValues::default()) + .unwrap(); + set + } + + fn specialize(num_vcpu: u8, max_vcpuid: Option, id: i32) -> CpuidSet { + let mut spec = Specializer::new() + .with_vcpu_count(NonZeroU8::new(num_vcpu).unwrap(), true) + .with_vcpuid(id) + .with_cpu_topo([TopoKind::Std4, TopoKind::StdB].into_iter()); + if let Some(max) = max_vcpuid { + spec = spec.with_max_vcpuid(max); + } + spec.execute(base_set()).unwrap() + } + + /// Returns (initial APIC ID from leaf 1, leaf B core-level shift, leaf B + /// x2APIC ID, number of APIC IDs sharing L3 per leaf 4). + fn topo(set: &CpuidSet) -> (u32, u32, u32, u32) { + let leaf1 = set.get(CpuidIdent::leaf(1)).unwrap(); + let leafb = set.get(CpuidIdent::subleaf(0xb, 1)).unwrap(); + let l3 = set.get(CpuidIdent::subleaf(4, 1)).unwrap(); + (leaf1.ebx >> 24, leafb.eax, leafb.edx, ((l3.eax >> 14) & 0xfff) + 1) + } + + #[test] + fn contiguous_ids_unchanged_by_max_vcpuid() { + // Specifying the max vCPU ID for a contiguous set of IDs must not + // change what guests see. + for id in 0..4 { + let implicit = specialize(4, None, id); + let explicit = specialize(4, Some(3), id); + implicit.is_equivalent_to(&explicit).unwrap(); + assert_eq!(topo(&explicit), (id as u32, 8, id as u32, 4)); + } + } + + #[test] + fn sparse_ids_span_all_apic_ids() { + for id in [0, 1, 256, 257] { + let set = specialize(4, Some(257), id); + let (initial_id, core_shift, x2apic_id, l3_sharing) = topo(&set); + + // Leaf 1 can only report the low 8 bits of the APIC ID. + assert_eq!(initial_id, id as u32 & 0xff); + assert_eq!(x2apic_id, id as u32); + + // All vCPUs must appear to be in the same package... + assert_eq!(core_shift, 9); + assert_eq!(x2apic_id >> core_shift, 0); + + // ...and share an L3 cache. + assert_eq!(l3_sharing, 258); + let l3_bits = u32::BITS - (l3_sharing - 1).leading_zeros(); + assert_eq!(x2apic_id >> l3_bits, 0); + } + } +} diff --git a/lib/propolis/src/firmware/acpi/madt.rs b/lib/propolis/src/firmware/acpi/madt.rs index ce0ee608b..c1bfd6dbf 100644 --- a/lib/propolis/src/firmware/acpi/madt.rs +++ b/lib/propolis/src/firmware/acpi/madt.rs @@ -21,13 +21,20 @@ const TIMER_GSI: u32 = 2; const LOCAL_APIC_INT_NUMBER: u8 = 1; +/// The xAPIC broadcast ID, which can't be assigned to a processor. APIC IDs at +/// or above this must be described with Processor Local x2APIC structures. +const XAPIC_BROADCAST_ID: u8 = 0xff; + /// Configuration for generating a MADT table. pub struct MadtConfig { /// The ACPI table variant to use. pub acpi_variant: AcpiVariant, - /// Number of vCPUs in the VM. - pub num_cpus: u8, + /// APIC IDs of the VM's vCPUs, in ascending order. + /// + /// Only APIC IDs below 255 are supported, since higher IDs require + /// Processor Local x2APIC structures, which aren't implemented yet. + pub apic_ids: Vec, } /// The MADT/APIC table describes the interrupts for the entire system. @@ -64,10 +71,14 @@ impl Aml for Madt { // Processor Local APIC. // ACPI rev. 6.6 section 5.2.12.2 "Processor Local APIC Structure" - for i in 0..self.config.num_cpus { + for &apic_id in self.config.apic_ids.iter() { + let apic_id = u8::try_from(apic_id) + .ok() + .filter(|&id| id != XAPIC_BROADCAST_ID) + .expect("APIC IDs >= 255 unsupported"); table.add_structure(madt::ProcessorLocalApic::new( - i, - i, + apic_id, + apic_id, madt::EnabledStatus::Enabled, )); } diff --git a/lib/propolis/src/hw/qemu/fwcfg.rs b/lib/propolis/src/hw/qemu/fwcfg.rs index 316f14b8a..715c3caf7 100644 --- a/lib/propolis/src/hw/qemu/fwcfg.rs +++ b/lib/propolis/src/hw/qemu/fwcfg.rs @@ -1316,7 +1316,8 @@ pub mod formats { /// Instance configuration that are relevant when building the ACPI tables. pub struct AcpiConfig<'a> { pub acpi_variant: acpi::AcpiVariant, - pub num_cpus: u8, + /// APIC IDs of the VM's vCPUs, in ascending order. + pub apic_ids: &'a [u32], pub pci_window_32: PciWindow, pub pci_window_64: PciWindow, pub dsdt_generators: &'a [&'a dyn acpi::DsdtGenerator], @@ -1591,7 +1592,7 @@ pub mod formats { fn add_madt(&mut self) -> usize { let madt_config = acpi::MadtConfig { acpi_variant: self.config.acpi_variant, - num_cpus: self.config.num_cpus, + apic_ids: self.config.apic_ids.to_vec(), }; let madt = acpi::Madt::new(madt_config); let madt_offset = self.tables.len(); diff --git a/lib/propolis/src/vmm/hdl.rs b/lib/propolis/src/vmm/hdl.rs index f34939bf7..c0f5b3a56 100644 --- a/lib/propolis/src/vmm/hdl.rs +++ b/lib/propolis/src/vmm/hdl.rs @@ -411,6 +411,17 @@ impl VmmHdl { unsafe { self.ioctl(bhyve_api::VM_SUSPEND, &mut data) } } + /// Query the number of vCPU IDs the kernel VMM supports for this VM. Valid + /// vCPU IDs are in `0..max_vcpus()`. + /// + /// bhyve allocates vCPUs lazily, by ID, on first use, so a VM may use any + /// (possibly sparse) set of IDs below this limit. + pub fn max_vcpus(&self) -> Result { + let mut topo = bhyve_api::vm_cpu_topology::default(); + unsafe { self.ioctl(bhyve_api::VM_GET_TOPOLOGY, &mut topo) }?; + Ok(topo.maxcpus) + } + pub fn reinit(&self, force_suspend: bool) -> Result<()> { let mut data = bhyve_api::vm_reinit { flags: 0 }; if force_suspend { diff --git a/lib/propolis/src/vmm/machine.rs b/lib/propolis/src/vmm/machine.rs index 36a150c8c..0d4f82377 100644 --- a/lib/propolis/src/vmm/machine.rs +++ b/lib/propolis/src/vmm/machine.rs @@ -68,6 +68,24 @@ impl Machine { self.vcpus.len() } + /// The highest vCPU ID in the machine. vCPU IDs may be sparse, so this + /// can exceed `num_vcpus() - 1`. + pub fn max_vcpu_id(&self) -> i32 { + self.vcpus.last().expect("machine has at least one vCPU").id + } + + /// APIC IDs of the machine's vCPUs, in ascending order. + /// + /// bhyve uses a vCPU's ID as its APIC ID, so these are the vCPU IDs. + pub fn apic_ids(&self) -> Vec { + self.vcpus + .iter() + .map(|vcpu| { + u32::try_from(vcpu.id).expect("vCPU IDs are non-negative") + }) + .collect() + } + /// (Re)Initialize vCPUs per x86 spec pub fn vcpu_x86_setup(&self) -> Result<()> { for vcpu in self.vcpus.iter() { @@ -196,7 +214,7 @@ impl Machine { pub struct Builder { inner_hdl: Option>, physmap: Option, - max_cpu: u8, + vcpu_ids: Vec, guest_hv_interface: Option>, } impl Builder { @@ -215,7 +233,7 @@ impl Builder { let physmap = Some(PhysMap::new(MAX_PHYSMEM, hdl.clone())); Ok(Self { inner_hdl: Some(hdl), - max_cpu: 1, + vcpu_ids: vec![0], guest_hv_interface: None, physmap, }) @@ -258,12 +276,46 @@ impl Builder { )?; Ok(self) } - /// Sets the maximum number of CPUs for the machine. + /// Sets the number of CPUs for the machine. The vCPUs are assigned + /// contiguous IDs starting from 0. pub fn max_cpus(mut self, max: u8) -> Result { if max == 0 || max > MAXCPU as u8 { Err(Error::new(ErrorKind::InvalidInput, "maxcpu out of range")) } else { - self.max_cpu = max; + self.vcpu_ids = (0..u16::from(max)).collect(); + Ok(self) + } + } + + /// Sets the IDs of the vCPUs to create for the machine. + /// + /// The IDs need not be contiguous. bhyve uses a vCPU's ID as its APIC ID, + /// so a sparse set of IDs (e.g. `[0, 1, 128, 129]`) lets a small VM + /// present APIC IDs that require x2APIC to address. + /// + /// The set must include ID 0 (the bootstrap processor) and may not contain + /// duplicates. Each ID must also be below the kernel VMM's per-VM limit, + /// which is checked by [`Builder::finalize`]. + pub fn vcpu_ids( + mut self, + ids: impl IntoIterator, + ) -> Result { + let mut ids: Vec = ids.into_iter().collect(); + ids.sort_unstable(); + let len = ids.len(); + ids.dedup(); + + if ids.len() != len { + Err(Error::new(ErrorKind::InvalidInput, "duplicate vCPU ID")) + } else if ids.is_empty() || ids.len() > MAXCPU { + Err(Error::new(ErrorKind::InvalidInput, "vCPU count out of range")) + } else if ids[0] != 0 { + Err(Error::new( + ErrorKind::InvalidInput, + "vCPU IDs must include BSP (0)", + )) + } else { + self.vcpu_ids = ids; Ok(self) } } @@ -280,6 +332,18 @@ impl Builder { /// Consumes `self` and creates a new [`Machine`] based /// on the provided memory regions. pub fn finalize(mut self) -> Result { + let max_id = *self.vcpu_ids.last().expect("vcpu_ids is non-empty"); + let vmm_maxcpu = self.inner_hdl.as_ref().unwrap().max_vcpus()?; + if max_id >= vmm_maxcpu { + return Err(Error::new( + ErrorKind::InvalidInput, + format!( + "vCPU ID {max_id} exceeds the kernel VMM's limit of \ + {vmm_maxcpu} vCPUs" + ), + )); + } + let hdl = self.inner_hdl.take().unwrap(); let mut map = self.physmap.take().unwrap(); @@ -294,8 +358,10 @@ impl Builder { .unwrap_or(Arc::new(BhyveGuestInterface::new())); guest_hv_interface.attach(&acc_mem, hdl.clone()); - let vcpus = (0..self.max_cpu) - .map(|id| { + let vcpus = self + .vcpu_ids + .iter() + .map(|&id| { Vcpu::new( hdl.clone(), i32::from(id), From 23620347e65d7ed81e623e5adc51651743a28e24 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Wed, 18 Nov 2026 17:24:38 -0800 Subject: [PATCH 8/8] propolis-standalone: allow vCPU IDs above 254 and generate x2APIC MADT entries for them --- bin/propolis-standalone/README.md | 6 +- bin/propolis-standalone/src/config.rs | 5 +- bin/propolis-standalone/src/main.rs | 8 -- lib/propolis/src/firmware/acpi/madt.rs | 146 +++++++++++++++++++++---- lib/propolis/src/vmm/machine.rs | 2 +- 5 files changed, 131 insertions(+), 36 deletions(-) diff --git a/bin/propolis-standalone/README.md b/bin/propolis-standalone/README.md index 2e3241dd4..95afbc38c 100644 --- a/bin/propolis-standalone/README.md +++ b/bin/propolis-standalone/README.md @@ -27,8 +27,10 @@ memory = 1024 # boot_order = ["net0", "block0"] # Assign specific (possibly sparse) IDs to the vCPUs, which bhyve also uses as -# their APIC IDs. There must be `cpus` entries, including 0 (default: 0..cpus) -# vcpu_ids = [0, 1, 128, 129] +# their APIC IDs. There must be `cpus` entries, including 0. IDs of 255 or +# above require advertising x2APIC in the CPUID profile and x2APIC-capable +# bootrom/guest. (default: 0..cpus) +# vcpu_ids = [0, 1, 256, 257] [block_dev.alpine_iso] type = "file" diff --git a/bin/propolis-standalone/src/config.rs b/bin/propolis-standalone/src/config.rs index e65422f3a..9612e95b1 100644 --- a/bin/propolis-standalone/src/config.rs +++ b/bin/propolis-standalone/src/config.rs @@ -53,9 +53,8 @@ pub struct Main { pub cpus: u8, /// IDs to assign to the VM's vCPUs. There must be `cpus` of them and they /// must include 0. bhyve uses a vCPU's ID as its APIC ID, so sparse IDs - /// (e.g. `[0, 1, 128, 129]`) can be used to test guest handling of - /// non-contiguous APIC IDs. IDs must currently be below 255, since higher - /// APIC IDs require x2APIC MADT entries, which aren't yet supported. + /// (e.g. `[0, 1, 256, 257]`) can be used to test guest handling of + /// non-contiguous APIC IDs or x2APIC. /// /// Default: `0..cpus` pub vcpu_ids: Option>, diff --git a/bin/propolis-standalone/src/main.rs b/bin/propolis-standalone/src/main.rs index 0ee651155..a67f76f05 100644 --- a/bin/propolis-standalone/src/main.rs +++ b/bin/propolis-standalone/src/main.rs @@ -1164,14 +1164,6 @@ fn setup_instance( ids.len() ); } - // bhyve uses vCPU IDs as APIC IDs, and the MADT can't yet - // describe APIC IDs that require x2APIC (propolis#956). - if let Some(id) = ids.iter().find(|&&id| id >= 0xff) { - anyhow::bail!( - "vcpu_ids contains {id}, but IDs >= 255 are not yet \ - supported" - ); - } ids.clone() } None => (0..u16::from(cpus)).collect(), diff --git a/lib/propolis/src/firmware/acpi/madt.rs b/lib/propolis/src/firmware/acpi/madt.rs index c1bfd6dbf..9efd7d4c4 100644 --- a/lib/propolis/src/firmware/acpi/madt.rs +++ b/lib/propolis/src/firmware/acpi/madt.rs @@ -12,6 +12,8 @@ use super::{ OEM_TABLE_ID, PCI_LINK_IRQS, }; use acpi_tables::{madt, Aml, AmlSink}; +use zerocopy::byteorder::little_endian::{U16, U32}; +use zerocopy::{Immutable, IntoBytes}; const IO_APIC_ID: u8 = 0x02; const IO_APIC_GSI_BASE: u32 = 0x0000; @@ -21,19 +23,24 @@ const TIMER_GSI: u32 = 2; const LOCAL_APIC_INT_NUMBER: u8 = 1; -/// The xAPIC broadcast ID, which can't be assigned to a processor. APIC IDs at -/// or above this must be described with Processor Local x2APIC structures. -const XAPIC_BROADCAST_ID: u8 = 0xff; +/// UID that refers to all processors in a Local xAPIC NMI structure. +const XAPIC_ALL_PROCESSORS_UID: u8 = 0xff; + +/// UID that refers to all processors in a Local x2APIC NMI structure. +const X2APIC_ALL_PROCESSORS_UID: u32 = 0xffff_ffff; + +/// Processors with APIC IDs at or above this value must be described with +/// the x2APIC variants of the Processor Local APIC structures. +/// +/// ACPI rev. 6.6 section 5.2.12.12 "Processor Local x2APIC Structure" +const X2APIC_ID_THRESHOLD: u32 = 0xff; /// Configuration for generating a MADT table. pub struct MadtConfig { /// The ACPI table variant to use. pub acpi_variant: AcpiVariant, - /// APIC IDs of the VM's vCPUs, in ascending order. - /// - /// Only APIC IDs below 255 are supported, since higher IDs require - /// Processor Local x2APIC structures, which aren't implemented yet. + /// (x2)APIC IDs of the VM's vCPUs, in ascending order. pub apic_ids: Vec, } @@ -69,18 +76,28 @@ impl Aml for Madt { ) .pc_at_compat(); - // Processor Local APIC. + // Processor Local APIC and Processor Local x2APIC. // ACPI rev. 6.6 section 5.2.12.2 "Processor Local APIC Structure" + // ACPI rev. 6.6 section 5.2.12.12 "Processor Local x2APIC Structure" + // + // Each processor's UID is its (x2)APIC ID. + let mut has_x2apic = false; for &apic_id in self.config.apic_ids.iter() { - let apic_id = u8::try_from(apic_id) - .ok() - .filter(|&id| id != XAPIC_BROADCAST_ID) - .expect("APIC IDs >= 255 unsupported"); - table.add_structure(madt::ProcessorLocalApic::new( - apic_id, - apic_id, - madt::EnabledStatus::Enabled, - )); + if apic_id < X2APIC_ID_THRESHOLD { + let apic_id = apic_id as u8; + table.add_structure(madt::ProcessorLocalApic::new( + apic_id, + apic_id, + madt::EnabledStatus::Enabled, + )); + } else { + has_x2apic = true; + table.add_structure(ProcessorLocalX2Apic::new( + apic_id, + apic_id, + madt::EnabledStatus::Enabled, + )); + } } // I/O APIC. @@ -106,16 +123,101 @@ impl Aml for Madt { ); }); - // Local APIC NMI. + // Local APIC NMI or Local x2APIC NMI. // ACPI rev. 6.6 section 5.2.12.7 "Local APIC NMI Structure" + // ACPI rev. 6.6 section 5.2.12.13 "Local x2APIC NMI Structure" + // + // Both apply to all processors via their "all processors" UIDs. If + // any processor has an x2APIC ID of 255 or greater, the global LINTx + // for every processor must be described with a Local x2APIC NMI + // structure. Otherwise, a Local APIC NMI structure is used. // // Supporting more than 255 vCPUs will require additional work. // Refer to propolis#956 for more information. - table.add_structure(madt::ProcessorLocalApicNmi::new( - 0xff, // Apply to all processors. - LOCAL_APIC_INT_NUMBER, - )); + if has_x2apic { + table.add_structure(LocalX2ApicNmi::new( + X2APIC_ALL_PROCESSORS_UID, + LOCAL_APIC_INT_NUMBER, + )); + } else { + table.add_structure(madt::ProcessorLocalApicNmi::new( + XAPIC_ALL_PROCESSORS_UID, + LOCAL_APIC_INT_NUMBER, + )); + } table.to_aml_bytes(sink); } } + +// XXX `acpi_tables` only provides the xAPIC flavors of these structures + +/// Processor Local x2APIC structure. +/// +/// ACPI rev. 6.6 section 5.2.12.12 "Processor Local x2APIC Structure" +#[repr(C, packed)] +#[derive(Clone, Copy, Debug, IntoBytes, Immutable)] +struct ProcessorLocalX2Apic { + r#type: u8, + length: u8, + _reserved: U16, + x2apic_id: U32, + flags: U32, + acpi_processor_uid: U32, +} + +impl ProcessorLocalX2Apic { + const TYPE: u8 = 0x9; + + fn new(uid: u32, x2apic_id: u32, enabled: madt::EnabledStatus) -> Self { + Self { + r#type: Self::TYPE, + length: std::mem::size_of::() as u8, + _reserved: 0.into(), + x2apic_id: x2apic_id.into(), + flags: (enabled as u32).into(), + acpi_processor_uid: uid.into(), + } + } +} + +impl Aml for ProcessorLocalX2Apic { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.vec(self.as_bytes()); + } +} + +/// Local x2APIC NMI structure. +/// +/// ACPI rev. 6.6 section 5.2.12.13 "Local x2APIC NMI Structure" +#[repr(C, packed)] +#[derive(Clone, Copy, Debug, IntoBytes, Immutable)] +struct LocalX2ApicNmi { + r#type: u8, + length: u8, + flags: U16, + acpi_processor_uid: U32, + local_x2apic_lint_num: u8, + _reserved: [u8; 3], +} + +impl LocalX2ApicNmi { + const TYPE: u8 = 0xa; + + fn new(acpi_processor_uid: u32, local_x2apic_lint_num: u8) -> Self { + Self { + r#type: Self::TYPE, + length: std::mem::size_of::() as u8, + flags: 0.into(), + acpi_processor_uid: acpi_processor_uid.into(), + local_x2apic_lint_num, + _reserved: [0; 3], + } + } +} + +impl Aml for LocalX2ApicNmi { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.vec(self.as_bytes()); + } +} diff --git a/lib/propolis/src/vmm/machine.rs b/lib/propolis/src/vmm/machine.rs index 0d4f82377..e99f245f6 100644 --- a/lib/propolis/src/vmm/machine.rs +++ b/lib/propolis/src/vmm/machine.rs @@ -290,7 +290,7 @@ impl Builder { /// Sets the IDs of the vCPUs to create for the machine. /// /// The IDs need not be contiguous. bhyve uses a vCPU's ID as its APIC ID, - /// so a sparse set of IDs (e.g. `[0, 1, 128, 129]`) lets a small VM + /// so a sparse set of IDs (e.g. `[0, 1, 256, 257]`) lets a small VM /// present APIC IDs that require x2APIC to address. /// /// The set must include ID 0 (the bootstrap processor) and may not contain