diff --git a/Cargo.lock b/Cargo.lock index 6b5e4b6db..6ecf7f8cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6107,6 +6107,7 @@ dependencies = [ "anyhow", "backoff", "byteorder", + "cfg-if", "chrono", "cpuid_utils", "dropshot 0.17.1", diff --git a/bin/propolis-cli/src/main.rs b/bin/propolis-cli/src/main.rs index 72ba175b4..6e93a0db1 100644 --- a/bin/propolis-cli/src/main.rs +++ b/bin/propolis-cli/src/main.rs @@ -17,10 +17,11 @@ use clap::{Args, Parser, Subcommand}; use futures::{future, SinkExt}; use newtype_uuid::{GenericUuid, TypedUuid, TypedUuidKind, TypedUuidTag}; use propolis_client::instance_spec::{ - BlobStorageBackend, Board, Chipset, Component, CrucibleStorageBackend, - I440Fx, InstanceMetadata, InstanceProperties, InstanceSpec, - InstanceSpecGetResponse, NvmeDisk, PciPath, QemuPvpanic, - ReplacementComponent, SerialPort, SerialPortNumber, SpecKey, VirtioDisk, + BlobStorageBackend, Board, Chipset, Component, CpuTopology, + CrucibleStorageBackend, I440Fx, InstanceMetadata, InstanceProperties, + InstanceSpec, InstanceSpecGetResponse, NvmeDisk, PciPath, QemuPvpanic, + ReplacementComponent, SerialPort, SerialPortNumber, SpecKey, TopoLeaves, + VirtioDisk, }; use propolis_client::support::nvme_serial_from_str; use propolis_client::types::{ @@ -174,7 +175,7 @@ struct VmConfig { /// Number of vCPUs allocated to instance #[clap(short = 'c', default_value = "4", action, requires = "config_toml")] - vcpus: u8, + vcpus: u16, /// Memory allocated to instance (MiB) #[clap(short, default_value = "1024", action, requires = "config_toml")] @@ -330,11 +331,31 @@ impl VmConfig { .map(|cfg| cfg.hv_interface.clone()) .unwrap_or_default(); + let smt = parsed_toml + .as_ref() + .map(|cfg| cfg.machine_settings.smt) + .unwrap_or(true); + + let sockets = parsed_toml + .as_ref() + .map(|cfg| cfg.machine_settings.sockets) + .unwrap_or(1); + let mut spec = InstanceSpec { board: Board { chipset: Chipset::I440Fx(I440Fx { enable_pcie }), cpuid: cpuid_profile, cpus: self.vcpus, + cpu_topology: CpuTopology { + has_smt: smt, + sockets, + topo_leaves: TopoLeaves { + std_4: true, + std_b: true, + ext_8: true, + ext_1e: false, + }, + }, memory_mb: self.memory, guest_hv_interface, }, diff --git a/bin/propolis-server/src/lib/initializer.rs b/bin/propolis-server/src/lib/initializer.rs index 0da580375..5110fe1f3 100644 --- a/bin/propolis-server/src/lib/initializer.rs +++ b/bin/propolis-server/src/lib/initializer.rs @@ -4,7 +4,7 @@ use std::convert::TryInto; use std::fs::File; -use std::num::{NonZeroU8, NonZeroUsize}; +use std::num::{NonZeroU16, NonZeroUsize}; use std::os::unix::fs::FileTypeExt; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; @@ -51,6 +51,7 @@ use propolis::intr_pins; use propolis::vmm::{self, Builder, Machine}; use propolis::vsock::GuestCid; use propolis_api_types::instance::InstanceProperties; +use propolis_api_types::instance_spec::components::board::TopoLeaves; use propolis_api_types::instance_spec::components::devices::SerialPortNumber; use propolis_api_types::instance_spec::{self, SpecKey}; use propolis_types::{CpuidIdent, CpuidVendor}; @@ -119,6 +120,9 @@ pub enum MachineInitError { #[error("failed to start attestation server")] AttestationServer(#[source] std::io::Error), + #[error("unsupported VM configuration")] + UnsupportedCpuConfig(&'static str), + #[cfg(feature = "falcon")] #[error("softnpu p9 device missing")] SoftNpuP9Missing, @@ -1220,7 +1224,7 @@ impl MachineInitializer<'_> { fn generate_smbios( &self, bootrom_version: &Option, - ) -> smbios::TableBytes { + ) -> Result { use smbios::table::{type0, type1, type16, type4}; let rom_size = @@ -1328,6 +1332,12 @@ impl MachineInitializer<'_> { .and_then(|vals| propolis::cpuid::parse_brand_string(vals).ok()) .unwrap_or_default(); + let num_cpus: u8 = self.spec.board.cpus.try_into().map_err(|_| { + MachineInitError::UnsupportedCpuConfig( + "more CPUs than expressible in Type 4 SMBIOS v2.7 tables", + ) + })?; + let smb_type4 = smbios::table::Type4 { proc_type: type4::ProcType::Central, proc_family, @@ -1338,9 +1348,9 @@ impl MachineInitializer<'_> { // unknown proc_upgrade: 0x2, // make core and thread counts equal for now - core_count: self.spec.board.cpus, - core_enabled: self.spec.board.cpus, - thread_count: self.spec.board.cpus, + core_count: num_cpus, + core_enabled: num_cpus, + thread_count: num_cpus, proc_characteristics: type4::Characteristics::IS_64_BIT | type4::Characteristics::MULTI_CORE, ..Default::default() @@ -1382,7 +1392,7 @@ impl MachineInitializer<'_> { smb_tables.add(0x1700.into(), &smb_type17).unwrap(); smb_tables.add(0x3200.into(), &smb_type32).unwrap(); - smb_tables.commit() + Ok(smb_tables.commit()) } fn generate_e820(&self) -> Result { @@ -1469,7 +1479,7 @@ impl MachineInitializer<'_> { fn generate_acpi_tables( &self, acpi_variant: acpi::AcpiVariant, - cpus: u8, + cpus: u16, ) -> Result { let (lowmem, _) = get_spec_guest_ram_limits(self.spec); let generators: Vec<_> = self @@ -1509,7 +1519,7 @@ impl MachineInitializer<'_> { /// Should not be called before [`Self::initialize_rom()`]. pub fn initialize_fwcfg( &mut self, - cpus: u8, + cpus: u16, bootrom_version: &Option, acpi_variant: acpi::AcpiVariant, ) -> Result, MachineInitError> { @@ -1522,7 +1532,7 @@ impl MachineInitializer<'_> { .map_err(|e| MachineInitError::FwcfgInsertFailed("cpu count", e))?; let smbios::TableBytes { entry_point, structure_table } = - self.generate_smbios(bootrom_version); + self.generate_smbios(bootrom_version)?; fwcfg .insert_named( "etc/smbios/smbios-tables", @@ -1580,11 +1590,49 @@ impl MachineInitializer<'_> { Ok(ramfb) } + fn topo_cpuid_leaves(&self) -> Result, MachineInitError> { + // This is trying to find a local minima of annoying.. the alternative + // here would to have `CpuTopology` hold a `Vec` directly. + // That opens questions like, what is the semantic of including a leaf + // multiple times? And we would want another glue type between the + // `propolis-server`-internal `TopoKind` that we use, to whatever + // version's external `TopoKind`. On those axes, it would be mildly + // simpler to just have a bag of bools for what leaves we want to + // specialize in this CPU topology. + // + // The downside of a bag of bools is that we have to map them to the + // internal `TopoKinds` enum at some point. This is that point. This + // conversion is incredibly typo-sensitive: would you spot + // `if std_4 { topo_leaves.push(TopoKind::StdB); }`? But converting + // across `TopoKinds` enums in versioned conversions would be similarly + // sensitive, and at least with a bag of booleans the fields are named + // and `std_4: std_b` would stand out. + let mut topo_leaves = Vec::new(); + + let TopoLeaves { std_4, std_b, ext_8, ext_1e } = + self.spec.board.cpu_topology.topo_leaves; + + if std_4 { + topo_leaves.push(TopoKind::Std4); + } + if std_b { + topo_leaves.push(TopoKind::StdB); + } + if ext_8 { + topo_leaves.push(TopoKind::Ext8); + } + if ext_1e { + topo_leaves.push(TopoKind::Ext1E); + } + Ok(topo_leaves) + } + /// Initialize virtual CPUs by first setting their capabilities, inserting /// them into the device map, and then, if a kstat sampler is provided, /// tracking their kstats. pub async fn initialize_cpus(&mut self) -> Result<(), MachineInitError> { let hv_interface = self.machine.guest_hv_interface.as_ref(); + for vcpu in self.machine.vcpus.iter() { // Report that the guest is running on bhyve. // @@ -1596,33 +1644,29 @@ impl MachineInitializer<'_> { requests to set hypervisor leaves", ); - // Instead of `TopoKind::supported`, we use an intentionally-reduced - // list of Intel-only leaves for the moment. This is because if we - // specialize leaves used by AMD (or just both vendors), we'll - // change the topology a guest sees. + let has_smt = self.spec.board.cpu_topology.has_smt; + // Instead of `TopoKind::supported`, the instance spec selects what + // leaves may get specialized topology data. This is to maintain the + // reported topology of VMs created either before CPU platforms or + // with platforms that inadvertently reported `board.cpus`-many + // single-core sockets. // - // The initial CPU platform defined in Nexus (Omicron#8728) hews to - // the pre-specialization topology, which won't have leaf B at all. - // Before that is sent, though, we'll see the present-but-zero - // leaves from bhyve, which we would happily specialize into - // something reflecting the guest if requested here. Once - // Omicron#8728 lands and propolis-server receives explicit CPUID - // profiles, we can add AMD leaves here too. - let cpu_topo_leaves = [TopoKind::Std4]; - - let specialized = propolis::cpuid::Specializer::new() - .with_vcpu_count( - NonZeroU8::new(self.spec.board.cpus).unwrap(), - true, - ) - .with_vcpuid(vcpu.id) - .with_cache_topo() - .clear_cpu_topo(TopoKind::iter()) - .with_cpu_topo(cpu_topo_leaves.into_iter()) - .execute(set) - .map_err(|e| { - MachineInitError::CpuidSpecializationFailed(vcpu.id, e) - })?; + // While it's probably OK to go change that, one would not expect + // their CPU sockets to unify across a reboot. + let cpu_topo_leaves = self.topo_cpuid_leaves()?; + info!(self.log, "computed cpu topo leaves {:?}", cpu_topo_leaves); + + let specialized = propolis::cpuid::Specializer::new( + NonZeroU16::new(self.spec.board.cpus).unwrap(), + ) + .with_cpu_topo(has_smt, cpu_topo_leaves.into_iter()) + .with_vcpuid(vcpu.id) + .with_cache_topo() + .clear_cpu_topo(TopoKind::iter()) + .execute(set) + .map_err(|e| { + MachineInitError::CpuidSpecializationFailed(vcpu.id, e) + })?; info!(self.log, "setting CPUID for vCPU"; "vcpu" => vcpu.id, diff --git a/bin/propolis-server/src/lib/migrate/types.rs b/bin/propolis-server/src/lib/migrate/types.rs index d88e24b15..2629d37a0 100644 --- a/bin/propolis-server/src/lib/migrate/types.rs +++ b/bin/propolis-server/src/lib/migrate/types.rs @@ -89,12 +89,14 @@ use serde::{Deserialize, Serialize}; use propolis_api_types_versions::v1::instance::ReplacementComponent; -use propolis_api_types_versions::{v1, v2, v3, v6}; +use propolis_api_types_versions::{v1, v2, v3, v6, v7}; use std::collections::BTreeMap; use crate::migrate::MigrateError; -use crate::spec::{api_spec_v1, api_spec_v2, api_spec_v3, api_spec_v6, Spec}; +use crate::spec::{ + api_spec_v1, api_spec_v2, api_spec_v3, api_spec_v6, api_spec_v7, Spec, +}; /// A wrapper for one of any supported `InstanceSpec` that describe a /// to-be-migrated VM. @@ -113,6 +115,7 @@ pub(crate) enum VersionedInstanceSpec { V2(v2::instance_spec::InstanceSpec), V3(v3::instance_spec::InstanceSpec), V6(v6::instance_spec::InstanceSpec), + V7(v7::instance_spec::InstanceSpec), } impl VersionedInstanceSpec { @@ -186,6 +189,13 @@ impl VersionedInstanceSpec { .map_err(|e| MigrateError::PreambleParse(e.to_string()))? .finish() } + VersionedInstanceSpec::V7(mut source_spec) => { + api_spec_v7::amend(&mut source_spec, replacements)?; + + api_spec_v7::v7_to_spec_builder(source_spec) + .map_err(|e| MigrateError::PreambleParse(e.to_string()))? + .finish() + } }; Ok(amended_spec) diff --git a/bin/propolis-server/src/lib/spec/api_spec_v1.rs b/bin/propolis-server/src/lib/spec/api_spec_v1.rs index 8440299ed..62cb971ab 100644 --- a/bin/propolis-server/src/lib/spec/api_spec_v1.rs +++ b/bin/propolis-server/src/lib/spec/api_spec_v1.rs @@ -9,10 +9,7 @@ use std::collections::BTreeMap; use propolis_api_types::instance_spec::{ - components::{ - board::Board as InstanceSpecBoard, - devices::{BootSettings, SerialPort as SerialPortDesc}, - }, + components::devices::{BootSettings, SerialPort as SerialPortDesc}, SpecKey, }; use propolis_api_types_versions::{ @@ -137,8 +134,14 @@ impl TryFrom for v1::instance_spec::InstanceSpec { spec.components.insert(key, val); } - let board = InstanceSpecBoard { - cpus: board.cpus, + let cpus_u8: u8 = board.cpus.try_into().map_err(|_| { + LegacyApiSpecError::IncompatibleComponent( + "v1 Board cannot express >255 vCPUs".to_string(), + ) + })?; + + let board = v1::components::board::Board { + cpus: cpus_u8, memory_mb: board.memory_mb, chipset: board.chipset, guest_hv_interface: board.guest_hv_interface, diff --git a/bin/propolis-server/src/lib/spec/api_spec_v2.rs b/bin/propolis-server/src/lib/spec/api_spec_v2.rs index 030f19e30..e75aac7b1 100644 --- a/bin/propolis-server/src/lib/spec/api_spec_v2.rs +++ b/bin/propolis-server/src/lib/spec/api_spec_v2.rs @@ -77,7 +77,7 @@ pub(crate) fn amend( #[cfg(test)] mod test { use propolis_api_types_versions::latest::components::board::{ - Board, Chipset, Cpuid, GuestHypervisorInterface, I440Fx, + Board, Chipset, CpuTopology, Cpuid, GuestHypervisorInterface, I440Fx, }; use propolis_api_types_versions::latest::{ self, @@ -108,6 +108,7 @@ mod test { entries: vec![], vendor: CpuidVendor::Amd, }), + cpu_topology: CpuTopology::single_socket_smt(), }, components: Default::default(), smbios: Some(smbios_input.clone()), diff --git a/bin/propolis-server/src/lib/spec/api_spec_v3.rs b/bin/propolis-server/src/lib/spec/api_spec_v3.rs index 3822e5e13..032e34df4 100644 --- a/bin/propolis-server/src/lib/spec/api_spec_v3.rs +++ b/bin/propolis-server/src/lib/spec/api_spec_v3.rs @@ -129,7 +129,7 @@ pub(crate) fn amend( #[cfg(test)] mod test { use propolis_api_types_versions::latest::components::board::{ - Board, Chipset, Cpuid, GuestHypervisorInterface, I440Fx, + Board, Chipset, CpuTopology, Cpuid, GuestHypervisorInterface, I440Fx, }; use propolis_api_types_versions::latest::{ self, @@ -154,6 +154,7 @@ mod test { entries: vec![], vendor: CpuidVendor::Amd, }), + cpu_topology: CpuTopology::single_socket_smt(), }, components: Default::default(), smbios: Some(SmbiosType1Input { diff --git a/bin/propolis-server/src/lib/spec/api_spec_v6.rs b/bin/propolis-server/src/lib/spec/api_spec_v6.rs index ac32e1e60..cb41a58bc 100644 --- a/bin/propolis-server/src/lib/spec/api_spec_v6.rs +++ b/bin/propolis-server/src/lib/spec/api_spec_v6.rs @@ -8,11 +8,13 @@ use std::collections::BTreeMap; use propolis_api_types::instance_spec::SpecKey; -use propolis_api_types_versions::{v1::instance::ReplacementComponent, v3, v6}; +use propolis_api_types_versions::{ + v1::instance::ReplacementComponent, v3, v6, v7, +}; use super::{builder::SpecBuilder, ApiSpecError, Disk, Spec, StorageDevice}; use crate::migrate::MigrateError; -use crate::spec::api_spec_latest; +use crate::spec::api_spec_v7; impl From for v6::instance_spec::InstanceSpec { fn from(mut val: Spec) -> Self { @@ -87,10 +89,14 @@ impl From for v6::instance_spec::InstanceSpec { pub(crate) fn v6_to_spec_builder( value: v6::instance_spec::InstanceSpec, ) -> Result { - api_spec_latest::latest_to_spec_builder(value) + let v7_spec: v7::instance_spec::InstanceSpec = value.into(); + api_spec_v7::v7_to_spec_builder(v7_spec) } -fn amend_component( +// pub(crate) as the next version (v7) still uses +// `v6::instance_spec::Component`, so this conversion is applicable in that +// module as well. +pub(crate) fn amend_component( id: &SpecKey, to_amend: &mut v6::instance_spec::Component, replacement: &ReplacementComponent, diff --git a/bin/propolis-server/src/lib/spec/api_spec_v7.rs b/bin/propolis-server/src/lib/spec/api_spec_v7.rs new file mode 100644 index 000000000..7bf989a98 --- /dev/null +++ b/bin/propolis-server/src/lib/spec/api_spec_v7.rs @@ -0,0 +1,67 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Conversions between [`propolis_api_types_versions::v7`] instance specs and +//! the internal [`super::Spec`] representation. + +use std::collections::BTreeMap; + +use propolis_api_types::instance_spec::SpecKey; +use propolis_api_types_versions::{v1::instance::ReplacementComponent, v6, v7}; + +use super::{builder::SpecBuilder, ApiSpecError, Spec}; +use crate::migrate::MigrateError; +use crate::spec::{api_spec_latest, api_spec_v6}; + +impl From for v7::instance_spec::InstanceSpec { + fn from(mut spec: Spec) -> Self { + let (v6_spec, topo) = { + // v7 adds a CpuTopology field to the instance's Board. Remove it, + // convert the rest, add it back. + let older_topo = + v7::instance_spec::CpuTopology::legacy_single_thread_sockets( + spec.board.cpus, + ); + let topo = + std::mem::replace(&mut spec.board.cpu_topology, older_topo); + let v6_spec: v6::instance_spec::InstanceSpec = spec.into(); + (v6_spec, topo) + }; + + eprintln!("ok, spec to v7 instance spec.."); + + let mut spec: v7::instance_spec::InstanceSpec = v6_spec.into(); + spec.board.cpu_topology = topo; + + spec + } +} + +/// Parses a v7 instance spec into a [`SpecBuilder`], validating component +/// names, PCI paths, and backend references along the way. Callers can add +/// additional (non-v7) components to the builder before calling `finish()`. +pub(crate) fn v7_to_spec_builder( + value: v7::instance_spec::InstanceSpec, +) -> Result { + api_spec_latest::latest_to_spec_builder(value) +} + +pub(crate) fn amend( + spec: &mut v7::instance_spec::InstanceSpec, + replacements: &BTreeMap, +) -> Result<(), MigrateError> { + for (id, replacement) in replacements { + let Some(to_amend) = spec.components.get_mut(id) else { + return Err(MigrateError::InstanceSpecsIncompatible(format!( + "replacement component {id} not in source spec", + ))); + }; + + // v7 only changes `Board`, so the components which need amending are still the old v6 + // forms. That means we can just reuse the corresponding v6 `amend_component()`.. + api_spec_v6::amend_component(id, to_amend, replacement)?; + } + + Ok(()) +} diff --git a/bin/propolis-server/src/lib/spec/builder.rs b/bin/propolis-server/src/lib/spec/builder.rs index e3290369e..641a71626 100644 --- a/bin/propolis-server/src/lib/spec/builder.rs +++ b/bin/propolis-server/src/lib/spec/builder.rs @@ -112,6 +112,7 @@ impl SpecBuilder { memory_mb: board.memory_mb, chipset: board.chipset, guest_hv_interface: board.guest_hv_interface, + cpu_topology: board.cpu_topology, acpi_variant: propolis::firmware::acpi::AcpiVariant::V0, }, cpuid, @@ -406,7 +407,7 @@ impl SpecBuilder { mod test { use propolis_api_types::instance_spec::components::{ backends::{BlobStorageBackend, VirtioNetworkBackend}, - board::{Chipset, GuestHypervisorInterface, I440Fx}, + board::{Chipset, CpuTopology, GuestHypervisorInterface, I440Fx}, devices::{VirtioDisk, VirtioNic}, }; use uuid::Uuid; @@ -422,6 +423,7 @@ mod test { chipset: Chipset::I440Fx(I440Fx { enable_pcie: false }), guest_hv_interface: GuestHypervisorInterface::Bhyve, acpi_variant: propolis::firmware::acpi::AcpiVariant::V0, + cpu_topology: CpuTopology::single_socket_smt(), }; SpecBuilder { diff --git a/bin/propolis-server/src/lib/spec/mod.rs b/bin/propolis-server/src/lib/spec/mod.rs index 79c613650..488917ee4 100644 --- a/bin/propolis-server/src/lib/spec/mod.rs +++ b/bin/propolis-server/src/lib/spec/mod.rs @@ -40,7 +40,7 @@ use propolis_api_types::instance_spec::{ BlobStorageBackend, CrucibleStorageBackend, FileStorageBackend, VirtioNetworkBackend, }, - board::{Chipset, GuestHypervisorInterface, I440Fx}, + board::{Chipset, CpuTopology, GuestHypervisorInterface, I440Fx}, devices::{ NvmeDisk, PciPciBridge, QemuPvpanic as QemuPvpanicDesc, SerialPortNumber, VirtioDisk, VirtioNic, @@ -70,6 +70,7 @@ pub(crate) mod api_spec_v1; pub(crate) mod api_spec_v2; pub(crate) mod api_spec_v3; pub(crate) mod api_spec_v6; +pub(crate) mod api_spec_v7; pub(crate) mod builder; /// An error that can arise in converting @@ -127,7 +128,7 @@ impl TryFrom for Spec { type Error = ApiSpecError; fn try_from(value: InstanceSpec) -> Result { - Ok(api_spec_v6::v6_to_spec_builder(value)?.finish()) + Ok(api_spec_v7::v7_to_spec_builder(value)?.finish()) } } @@ -198,10 +199,11 @@ pub(crate) struct Spec { /// [instance spec `Board`]: propolis_api_types::instance_spec::components::board::Board #[derive(Clone, Debug)] pub(crate) struct Board { - pub cpus: u8, + pub cpus: u16, pub memory_mb: u64, pub chipset: Chipset, pub guest_hv_interface: GuestHypervisorInterface, + pub cpu_topology: CpuTopology, // XXX: expose via the API once more variants are implemented. pub acpi_variant: AcpiVariant, @@ -214,6 +216,7 @@ impl Default for Board { memory_mb: 0, chipset: Chipset::I440Fx(I440Fx { enable_pcie: false }), guest_hv_interface: GuestHypervisorInterface::Bhyve, + cpu_topology: CpuTopology::single_socket_smt(), acpi_variant: AcpiVariant::V0, } } diff --git a/bin/propolis-server/src/lib/stats/virtual_machine.rs b/bin/propolis-server/src/lib/stats/virtual_machine.rs index 976c11031..781ecd2bf 100644 --- a/bin/propolis-server/src/lib/stats/virtual_machine.rs +++ b/bin/propolis-server/src/lib/stats/virtual_machine.rs @@ -63,7 +63,7 @@ pub struct VirtualMachine { impl VirtualMachine { pub fn new( - n_vcpus: u8, + n_vcpus: u16, properties: &propolis_api_types::instance::InstanceProperties, ) -> Self { Self { diff --git a/bin/propolis-standalone/src/config.rs b/bin/propolis-standalone/src/config.rs index 6a0d24353..2b7d6a260 100644 --- a/bin/propolis-standalone/src/config.rs +++ b/bin/propolis-standalone/src/config.rs @@ -47,10 +47,25 @@ impl Config { } } +fn default_smt() -> bool { + true +} + +fn default_sockets() -> u16 { + 1 +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Main { pub name: String, - pub cpus: u8, + pub cpus: u16, + + // TODO: I'd *really* like to know what the Hygon 4-way SMT CPUID leaves look like.. + #[serde(default = "default_smt")] + pub smt: bool, + #[serde(default = "default_sockets")] + pub sockets: u16, + pub bootrom: String, pub bootrom_version: Option, pub memory: usize, diff --git a/bin/propolis-standalone/src/main.rs b/bin/propolis-standalone/src/main.rs index 57c290331..7eeb0b1d9 100644 --- a/bin/propolis-standalone/src/main.rs +++ b/bin/propolis-standalone/src/main.rs @@ -778,7 +778,7 @@ impl Instance { fn build_machine( log: &slog::Logger, name: &str, - max_cpu: u8, + max_cpu: u16, lowmem: usize, highmem: usize, use_reservoir: bool, @@ -902,7 +902,7 @@ struct SmbiosParams { memory_size: usize, rom_size: usize, rom_version: String, - num_cpus: u8, + num_cpus: u16, cpuid_ident: Option, cpuid_procname: Option<[CpuidValues; 3]>, } @@ -975,6 +975,10 @@ fn generate_smbios(params: SmbiosParams) -> anyhow::Result { .and_then(|e| cpuid::parse_brand_string(e).ok()) .unwrap_or("".to_string()); + let num_cpus: u8 = params.num_cpus.try_into().map_err(|_| { + anyhow::anyhow!("SMBIOS tables cannot yet express >255 CPUs") + })?; + let smb_type4 = smbios::table::Type4 { proc_type: type4::ProcType::Central, proc_family, @@ -982,12 +986,14 @@ fn generate_smbios(params: SmbiosParams) -> anyhow::Result { proc_id, proc_version: proc_version.as_str().try_into().unwrap_or_default(), status: type4::ProcStatus::Enabled, - // unknown + // "Processor Upgrade" of 2 means "Unknown" as in "Unknown upgrade + // method". This is somewhat more appropriate than any other value, + // which all suggest a socket is involved. proc_upgrade: 0x2, - // make core and thread counts equal for now - core_count: params.num_cpus, - core_enabled: params.num_cpus, - thread_count: params.num_cpus, + core_count: num_cpus, + core_enabled: num_cpus, + // TODO: fill in thread count from smt + thread_count: num_cpus, proc_characteristics: type4::Characteristics::IS_64_BIT | type4::Characteristics::MULTI_CORE, ..Default::default() @@ -1103,7 +1109,7 @@ fn generate_bootorder( fn generate_acpi_tables( acpi_variant: acpi::AcpiVariant, - num_cpus: u8, + num_cpus: u16, lowmem: usize, inventory: &Inventory, device_metadata: &DeviceMetadataMap, @@ -1584,18 +1590,27 @@ fn setup_instance( .add_cpuid(&mut base_profile) .context("failed to add hypervisor cpuid leaves")?; + let cpus = config.main.cpus; + + if !cpus.is_multiple_of(config.main.sockets) { + anyhow::bail!( + "vCPUs do not evenly divide into sockets: {} % {} != 0", + cpus, + config.main.sockets + ); + } + let cpus_per_socket = cpus / config.main.sockets; + for vcpu in machine.vcpus.iter() { - let vcpu_profile = propolis::cpuid::Specializer::new() - .with_vcpu_count( - std::num::NonZeroU8::new(config.main.cpus).unwrap(), - true, - ) - .with_vcpuid(vcpu.id) - .with_cache_topo() - .clear_cpu_topo(cpuid::TopoKind::iter()) - .with_cpu_topo(cpuid::TopoKind::supported()) - .execute(base_profile.clone()) - .context("failed to specialize cpuid profile")?; + let vcpu_profile = propolis::cpuid::Specializer::new( + std::num::NonZeroU16::new(cpus_per_socket).unwrap(), + ) + .with_vcpuid(vcpu.id) + .with_cache_topo() + .clear_cpu_topo(cpuid::TopoKind::iter()) + .with_cpu_topo(config.main.smt, cpuid::TopoKind::supported()) + .execute(base_profile.clone()) + .context("failed to specialize cpuid profile")?; vcpu.set_cpuid(vcpu_profile)?; vcpu.set_default_capabs()?; diff --git a/crates/propolis-api-types-versions/src/cpu_topology/api.rs b/crates/propolis-api-types-versions/src/cpu_topology/api.rs new file mode 100644 index 000000000..553f7cfba --- /dev/null +++ b/crates/propolis-api-types-versions/src/cpu_topology/api.rs @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! API request and response types for the TODO API version. + +use std::{collections::BTreeMap, net::SocketAddr}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::instance_spec::InstanceSpec; +use crate::v1::instance::{InstanceProperties, ReplacementComponent}; +use crate::v1::instance_spec::SpecKey; +use crate::v6; + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(tag = "method", content = "value")] +pub enum InstanceInitializationMethod { + Spec { + spec: InstanceSpec, + }, + MigrationTarget { + migration_id: Uuid, + src_addr: SocketAddr, + replace_components: BTreeMap, + }, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InstanceEnsureRequest { + pub properties: InstanceProperties, + pub init: InstanceInitializationMethod, +} + +impl From + for InstanceInitializationMethod +{ + fn from(old: v6::api::InstanceInitializationMethod) -> Self { + match old { + v6::api::InstanceInitializationMethod::Spec { spec } => { + Self::Spec { spec: spec.into() } + } + v6::api::InstanceInitializationMethod::MigrationTarget { + migration_id, + src_addr, + replace_components, + } => Self::MigrationTarget { + migration_id, + src_addr, + replace_components, + }, + } + } +} + +impl From for InstanceEnsureRequest { + fn from(old: v6::api::InstanceEnsureRequest) -> Self { + Self { properties: old.properties, init: old.init.into() } + } +} diff --git a/crates/propolis-api-types-versions/src/cpu_topology/components/board.rs b/crates/propolis-api-types-versions/src/cpu_topology/components/board.rs new file mode 100644 index 000000000..faf3da9d3 --- /dev/null +++ b/crates/propolis-api-types-versions/src/cpu_topology/components/board.rs @@ -0,0 +1,91 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! VM mainboard components. Every VM has a board, even if it has no other +//! peripherals. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v1::components::board::{Chipset, Cpuid, GuestHypervisorInterface}; + +/// A collection of CPUID leaves to specialize for a board's `CpuTopology`. +/// +/// Leaves may be selected here without being present in `Cpuid`, in which case +/// the extra selected topology leaves are ignored. Conversely, `Cpuid` leaves +/// with topology information that are not mentioend here probably indicate a +/// bug (at best, those leaves will probably indicate collection of single-core +/// sockets in the machine.) +#[derive(Copy, Clone, PartialEq, Deserialize, Serialize, Debug, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TopoLeaves { + pub std_4: bool, + pub std_b: bool, + pub ext_8: bool, + pub ext_1e: bool, +} + +/// A *very* simple topology for for a VM's vCPUs. +/// +/// Typically this will be configured as `threads_per_core: 2` to indicate SMT, +/// and one socket. The number of vCPUs in each socket is the same, meaning that +/// the number of vCPUs must divide evenly into the number of sockets in the +/// topology. +// TODO: Should this express a NUMA topology as well? Probably! Though an +// explicit topology would make migration trickier.. +// TODO: How does this describe CPU bindings? Should it? Probably. Nexus will +// eventually make decisions about what VM's vCPUs are placed where on sleds, +// and we'll want to faithfully indicate information about what processors are +// physically adjacent .... *probably*; doing that it harder to migrate without +// starting to lie. +#[derive(Copy, Clone, PartialEq, Deserialize, Serialize, Debug, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuTopology { + pub has_smt: bool, + pub sockets: u16, + // One might imagine TopoLeaves on either `Cpuid` or here, with topology information. I settled + // on Board->CpuTopology->TopoLeaves because CpuTopology ultimately decides aspects like "how + // many sockets does the guest think this board has", so the topology leaves we specialize are + // more "part of the board" than "part of the CPUs going in the board". Since a reasonable + // interpretation of CPUID leaves are that they come "from" a CPU in a socket, it is a little + // funky that these aren't part of `Cpuid`. + // + // Metaphysically speaking, there are kind of `MAX_CPUS * permutations(TopoLeaves)` distinct + // "CPU SKUs" that could be present, so what's really important is that the specialization + // `(Cpuid, vcpus, CpuTopology) -> CpuidSet` is stable. + pub topo_leaves: TopoLeaves, +} + +/// A VM's mainboard. +#[derive(Clone, Deserialize, Serialize, Debug, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Board { + /// The number of virtual logical processors attached to this VM. + pub cpus: u16, + + /// The amount of guest RAM attached to this VM. + pub memory_mb: u64, + + /// The chipset to expose to guest software. + pub chipset: Chipset, + + /// The hypervisor platform to expose to the guest. The default is a + /// bhyve-compatible interface with no additional features. + /// + /// For compatibility with older versions of Propolis, this field is only + /// serialized if it specifies a non-default interface. + #[serde( + default, + skip_serializing_if = "GuestHypervisorInterface::is_default" + )] + pub guest_hv_interface: GuestHypervisorInterface, + + /// The CPUID values to expose to the guest. If `None`, bhyve will derive + /// default values from the host's CPUID values. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpuid: Option, + + /// A *very* simplistic description of this VM's CPU topology. + pub cpu_topology: CpuTopology, +} diff --git a/crates/propolis-api-types-versions/src/cpu_topology/components/mod.rs b/crates/propolis-api-types-versions/src/cpu_topology/components/mod.rs new file mode 100644 index 000000000..ee9226076 --- /dev/null +++ b/crates/propolis-api-types-versions/src/cpu_topology/components/mod.rs @@ -0,0 +1,5 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +pub mod board; diff --git a/crates/propolis-api-types-versions/src/cpu_topology/instance_spec.rs b/crates/propolis-api-types-versions/src/cpu_topology/instance_spec.rs new file mode 100644 index 000000000..c575c6901 --- /dev/null +++ b/crates/propolis-api-types-versions/src/cpu_topology/instance_spec.rs @@ -0,0 +1,111 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v1::instance::{InstanceProperties, InstanceState}; +use crate::v1::instance_spec::SpecKey; +use crate::v2::instance_spec::SmbiosType1Input; +use crate::v6; +use crate::v6::instance_spec::Component as V6Component; + +pub use super::components::board::{Board, CpuTopology}; + +#[derive(Clone, Deserialize, Serialize, Debug, JsonSchema)] +pub struct InstanceSpec { + pub board: Board, + pub components: BTreeMap, + pub smbios: Option, +} + +#[derive(Clone, Deserialize, Serialize, JsonSchema)] +#[serde(tag = "type", content = "value")] +pub enum InstanceSpecStatus { + WaitingForMigrationSource, + Present(InstanceSpec), +} + +#[derive(Clone, Deserialize, Serialize, JsonSchema)] +pub struct InstanceSpecGetResponse { + pub properties: InstanceProperties, + pub state: InstanceState, + pub spec: InstanceSpecStatus, +} + +impl From for v6::instance_spec::InstanceSpec { + fn from(new: InstanceSpec) -> Self { + let Board { + cpus, + memory_mb, + chipset, + guest_hv_interface, + cpuid, + // This `From` impl, like others, is lossy. It should probably be a + // try_from and fail if this is not the implied default topology in + // v6. + cpu_topology: _, + } = new.board; + + // Like above, this is lossy, should instead be an error in a TryFrom. + let clamped_cpus = std::cmp::min(cpus, u8::MAX as u16) as u8; + + let old_board = crate::v1::components::board::Board { + cpus: clamped_cpus, + memory_mb, + chipset, + guest_hv_interface, + cpuid, + }; + + Self { + board: old_board, + components: new.components, + smbios: new.smbios, + } + } +} + +impl From for v6::instance_spec::InstanceSpecStatus { + fn from(new: InstanceSpecStatus) -> Self { + match new { + InstanceSpecStatus::WaitingForMigrationSource => { + Self::WaitingForMigrationSource + } + InstanceSpecStatus::Present(spec) => Self::Present(spec.into()), + } + } +} + +impl From + for v6::instance_spec::InstanceSpecGetResponse +{ + fn from(new: InstanceSpecGetResponse) -> Self { + Self { + properties: new.properties, + state: new.state, + spec: new.spec.into(), + } + } +} + +impl From for InstanceSpec { + fn from(old: v6::instance_spec::InstanceSpec) -> Self { + let cpus = u16::from(old.board.cpus); + Self { + board: Board { + cpus, + memory_mb: old.board.memory_mb, + chipset: old.board.chipset, + guest_hv_interface: old.board.guest_hv_interface, + cpuid: old.board.cpuid, + cpu_topology: CpuTopology::legacy_single_thread_sockets(cpus), + }, + components: old.components, + smbios: old.smbios, + } + } +} diff --git a/crates/propolis-api-types-versions/src/cpu_topology/mod.rs b/crates/propolis-api-types-versions/src/cpu_topology/mod.rs new file mode 100644 index 000000000..29cf1e870 --- /dev/null +++ b/crates/propolis-api-types-versions/src/cpu_topology/mod.rs @@ -0,0 +1,12 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Version `NVME_WRITE_CACHE` of the Propolis Server API. +//! +//! This version adds an API field to control if an NVMe device reports having a +//! volatile write cache. + +pub mod api; +pub mod components; +pub mod instance_spec; diff --git a/crates/propolis-api-types-versions/src/impls/instance_spec.rs b/crates/propolis-api-types-versions/src/impls/instance_spec.rs index 92100610f..ae3d41334 100644 --- a/crates/propolis-api-types-versions/src/impls/instance_spec.rs +++ b/crates/propolis-api-types-versions/src/impls/instance_spec.rs @@ -4,6 +4,8 @@ //! Functional code for instance spec types. +mod components; + use uuid::Uuid; use crate::latest::instance_spec::SpecKey; diff --git a/crates/propolis-api-types-versions/src/impls/instance_spec/components.rs b/crates/propolis-api-types-versions/src/impls/instance_spec/components.rs new file mode 100644 index 000000000..6834f4497 --- /dev/null +++ b/crates/propolis-api-types-versions/src/impls/instance_spec/components.rs @@ -0,0 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Functional code for component types. + +mod board; diff --git a/crates/propolis-api-types-versions/src/impls/instance_spec/components/board.rs b/crates/propolis-api-types-versions/src/impls/instance_spec/components/board.rs new file mode 100644 index 000000000..c4af34677 --- /dev/null +++ b/crates/propolis-api-types-versions/src/impls/instance_spec/components/board.rs @@ -0,0 +1,50 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::latest::components::board::{CpuTopology, TopoLeaves}; + +impl CpuTopology { + /// A `CpuTopology` describing the initial (and unintended) structure of + /// "single-thread-per-socket" for a VM. + /// + /// This further implies vCPU-many single-thread sockets, which many guests + /// either begrudgingly tolerate or worse particularly for high socket + /// counts. The worst cases here are OSes with fixed 64-entry arrays where a + /// "socket 65" or above causes stray memory accesses! This constructor is + /// exclusively to support VMs on the "initial" virtual platform, in + /// contrast to the "generic" virtual platform with a more normal + /// 2-threads-per-core single-socket topology. + pub fn legacy_single_thread_sockets(sockets: u16) -> Self { + Self { + has_smt: true, + sockets, + // The initial CPU platform as presented was even funkier than + // simply "vCPU-many single-thread sockets"; we would report SMT + // (above), but clear topology information from CPUID leaves Bh and + // 8000_001Eh. CPUID leaf 4h got topology information because it is + // Intel-only, so by populating that information we would not alter + // VMs in the product but would get to make Intel more right by + // default. + topo_leaves: TopoLeaves { + std_4: true, + std_b: false, + ext_8: false, + ext_1e: false, + }, + } + } + + pub fn single_socket_smt() -> Self { + Self { + has_smt: true, + sockets: 1, + topo_leaves: TopoLeaves { + std_4: true, + std_b: true, + ext_8: true, + ext_1e: true, + }, + } + } +} diff --git a/crates/propolis-api-types-versions/src/latest.rs b/crates/propolis-api-types-versions/src/latest.rs index 5e64474b0..5ea42f468 100644 --- a/crates/propolis-api-types-versions/src/latest.rs +++ b/crates/propolis-api-types-versions/src/latest.rs @@ -17,13 +17,16 @@ pub mod components { } pub mod board { - pub use crate::v1::components::board::Board; pub use crate::v1::components::board::Chipset; pub use crate::v1::components::board::Cpuid; pub use crate::v1::components::board::CpuidEntry; pub use crate::v1::components::board::GuestHypervisorInterface; pub use crate::v1::components::board::HyperVFeatureFlag; pub use crate::v1::components::board::I440Fx; + + pub use crate::v7::components::board::Board; + pub use crate::v7::components::board::CpuTopology; + pub use crate::v7::components::board::TopoLeaves; } pub mod devices { @@ -71,8 +74,8 @@ pub mod instance { pub use crate::v1::instance::InstanceStateRequested; pub use crate::v1::instance::ReplacementComponent; - pub use crate::v6::api::InstanceEnsureRequest; - pub use crate::v6::api::InstanceInitializationMethod; + pub use crate::v7::api::InstanceEnsureRequest; + pub use crate::v7::api::InstanceInitializationMethod; } pub mod instance_spec { @@ -86,9 +89,10 @@ pub mod instance_spec { pub use crate::v2::instance_spec::SmbiosType1Input; pub use crate::v6::instance_spec::Component; - pub use crate::v6::instance_spec::InstanceSpec; - pub use crate::v6::instance_spec::InstanceSpecGetResponse; - pub use crate::v6::instance_spec::InstanceSpecStatus; + + pub use crate::v7::instance_spec::InstanceSpec; + pub use crate::v7::instance_spec::InstanceSpecGetResponse; + pub use crate::v7::instance_spec::InstanceSpecStatus; } pub mod migration { diff --git a/crates/propolis-api-types-versions/src/lib.rs b/crates/propolis-api-types-versions/src/lib.rs index aecc1a645..858e8c388 100644 --- a/crates/propolis-api-types-versions/src/lib.rs +++ b/crates/propolis-api-types-versions/src/lib.rs @@ -41,3 +41,5 @@ pub mod v3; pub mod v5; #[path = "nvme_write_cache/mod.rs"] pub mod v6; +#[path = "cpu_topology/mod.rs"] +pub mod v7; diff --git a/crates/propolis-config-toml/src/lib.rs b/crates/propolis-config-toml/src/lib.rs index f98820ff6..ec370c23f 100644 --- a/crates/propolis-config-toml/src/lib.rs +++ b/crates/propolis-config-toml/src/lib.rs @@ -69,6 +69,14 @@ pub enum HypervisorInterface { }, } +fn default_smt() -> bool { + true +} + +fn default_sockets() -> u16 { + 1 +} + /// Settings covering the VM "at large". /// /// This corresponds to (and is a subset of) the `main` block understood by @@ -118,6 +126,11 @@ pub struct MachineSettings { // // That all to say: this could be a list. pub hv_interface: Option, + + #[serde(default = "default_smt")] + pub smt: bool, + #[serde(default = "default_sockets")] + pub sockets: u16, } /// The instance's chipset. diff --git a/crates/propolis-server-api/src/lib.rs b/crates/propolis-server-api/src/lib.rs index 298d0d33e..a147c2adc 100644 --- a/crates/propolis-server-api/src/lib.rs +++ b/crates/propolis-server-api/src/lib.rs @@ -8,7 +8,7 @@ use dropshot::{ WebsocketChannelResult, WebsocketConnection, }; use dropshot_api_manager_types::api_versions; -use propolis_api_types_versions::{latest, v1, v2, v3, v6}; +use propolis_api_types_versions::{latest, v1, v2, v3, v6, v7}; api_versions!([ // WHEN CHANGING THE API (part 1 of 2): @@ -22,6 +22,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (7, CPU_TOPOLOGY), (6, NVME_WRITE_CACHE), (5, CRUCIBLE_VOLUME_INFO), (4, DROPSHOT_BUMP_WEBSOCKET), @@ -49,7 +50,7 @@ pub trait PropolisServerApi { #[endpoint { method = PUT, path = "/instance", - versions = VERSION_NVME_WRITE_CACHE.. + versions = VERSION_CPU_TOPOLOGY.. }] async fn instance_ensure( rqctx: RequestContext, @@ -59,6 +60,26 @@ pub trait PropolisServerApi { HttpError, >; + #[endpoint { + operation_id = "instance_ensure", + method = PUT, + path = "/instance", + versions = VERSION_NVME_WRITE_CACHE..VERSION_CPU_TOPOLOGY + }] + async fn instance_ensure_v6( + rqctx: RequestContext, + request: TypedBody, + ) -> Result< + HttpResponseCreated, + HttpError, + > { + Self::instance_ensure( + rqctx, + request.map(v7::api::InstanceEnsureRequest::from), + ) + .await + } + #[endpoint { operation_id = "instance_ensure", method = PUT, @@ -72,7 +93,7 @@ pub trait PropolisServerApi { HttpResponseCreated, HttpError, > { - Self::instance_ensure( + Self::instance_ensure_v6( rqctx, request.map(v6::api::InstanceEnsureRequest::from), ) @@ -122,7 +143,7 @@ pub trait PropolisServerApi { #[endpoint { method = GET, path = "/instance/spec", - versions = VERSION_NVME_WRITE_CACHE.. + versions = VERSION_CPU_TOPOLOGY.. }] async fn instance_spec_get( rqctx: RequestContext, @@ -131,6 +152,22 @@ pub trait PropolisServerApi { HttpError, >; + #[endpoint { + method = GET, + path = "/instance/spec", + versions = VERSION_NVME_WRITE_CACHE..VERSION_CPU_TOPOLOGY + }] + async fn instance_spec_get_v6( + rqctx: RequestContext, + ) -> Result< + HttpResponseOk, + HttpError, + > { + Ok(Self::instance_spec_get(rqctx) + .await? + .map(v6::instance_spec::InstanceSpecGetResponse::from)) + } + #[endpoint { operation_id = "instance_spec_get", method = GET, @@ -143,7 +180,7 @@ pub trait PropolisServerApi { HttpResponseOk, HttpError, > { - Ok(Self::instance_spec_get(rqctx) + Ok(Self::instance_spec_get_v6(rqctx) .await? .map(v3::instance_spec::InstanceSpecGetResponse::from)) } diff --git a/lib/propolis/src/cpuid.rs b/lib/propolis/src/cpuid.rs index 7ec98276e..5c78d02b4 100644 --- a/lib/propolis/src/cpuid.rs +++ b/lib/propolis/src/cpuid.rs @@ -7,7 +7,7 @@ use std::cmp::Ordering; use std::collections::BTreeMap; use std::collections::BTreeSet; -use std::num::NonZeroU8; +use std::num::NonZeroU16; use std::ops::Bound; use bhyve_api::vcpu_cpuid_entry; @@ -45,36 +45,36 @@ pub enum SpecializeError { MissingVcpuCount, #[error("missing vcpu id")] MissingVcpuId, - #[error("unable to specialize leaf")] - IncompatibleTopology { leaf: u32, num_vcpu: u32, why: Option<&'static str> }, + #[error("unable to specialize leaf {leaf:08x}, {why:?} value={value:x}")] + IncompatibleTopology { leaf: u32, value: u32, why: Option<&'static str> }, } /// Specialize a set of cpuid leafs for provided attributes. /// /// This includes things such as a CPU topology (cores/threads/etc), a given /// vCPU ID (APIC, core/thread ID, etc), or other info tidbits. -#[derive(Default)] pub struct Specializer { - has_smt: bool, - num_vcpu: Option, + num_vcpu: NonZeroU16, vcpuid: Option, + has_smt: bool, cpu_topo_populate: BTreeSet, cpu_topo_clear: BTreeSet, do_cache_topo: bool, } impl Specializer { - pub fn new() -> Self { - Self::default() - } - - /// Specify number of vCPUs in instance, and if SMT is enabled - pub fn with_vcpu_count(self, count: NonZeroU8, has_smt: bool) -> Self { - Self { num_vcpu: Some(count), has_smt, ..self } + pub fn new(count: NonZeroU16) -> Self { + Self { + num_vcpu: count, + vcpuid: None, + has_smt: false, + cpu_topo_populate: BTreeSet::new(), + cpu_topo_clear: BTreeSet::new(), + do_cache_topo: false, + } } /// Specify vCPU ID to specialize for pub fn with_vcpuid(self, vcpuid: i32) -> Self { - assert!((vcpuid as usize) < crate::vcpu::MAXCPU); Self { vcpuid: Some(vcpuid), ..self } } @@ -85,6 +85,7 @@ impl Specializer { /// information may be incomplete. pub fn with_cpu_topo( self, + has_smt: bool, populate: impl Iterator, ) -> Self { let mut cpu_topo_populate = BTreeSet::new(); @@ -93,7 +94,7 @@ impl Specializer { cpu_topo_populate.insert(t); } - Self { cpu_topo_populate, ..self } + Self { has_smt, cpu_topo_populate, ..self } } /// Specify CPU topology types to clear from the specialized [CpuidSet] @@ -124,7 +125,7 @@ impl Specializer { ) -> Result { match set.vendor() { CpuidVendor::Amd => { - if self.do_cache_topo && self.num_vcpu.is_some() { + if self.do_cache_topo { self.fix_amd_cache_topo(&mut set)?; } } @@ -136,22 +137,54 @@ impl Specializer { // APIC ID based on vcpuid if let Some(vcpuid) = self.vcpuid.as_ref() { + if *vcpuid as usize >= crate::vcpu::MAXCPU { + return Err(SpecializeError::IncompatibleTopology { + leaf: 0, + value: *vcpuid as u32, + why: Some( + "More vCPUs than MAXCPU? This should have been checked \ + when creating the Machine.", + ), + }); + } + if let Some(ent) = set.get_mut(CpuidIdent::leaf(1)) { // bits 31:24 contain initial APIC ID ent.ebx &= !0xff000000; - ent.ebx |= ((*vcpuid as u32) & 0xff) << 24; + + let apic_id: u8 = (*vcpuid).try_into().map_err(|_| { + SpecializeError::IncompatibleTopology { + leaf: 1, + value: *vcpuid as u32, + why: Some( + "Don't know how to set CPUID initial APIC ID \ + if there are more than 255 IDs!", + ), + } + })?; + ent.ebx |= (apic_id as u32 & 0xff) << 24; } } // logical CPU count (if SMT is enabled) - if let Some(num_vcpu) = self.num_vcpu.as_ref() { - if self.has_smt { - if let Some(ent) = set.get_mut(CpuidIdent::leaf(1)) { - ent.edx |= (0x1 << 28); - // bits 23:16 contain max IDs for logical CPUs in package - ent.ebx &= !0xff0000; - ent.ebx |= u32::from(num_vcpu.get()) << 16; - } + if self.has_smt { + if let Some(ent) = set.get_mut(CpuidIdent::leaf(1)) { + ent.edx |= (0x1 << 28); + // bits 23:16 contain max IDs for logical CPUs in package + // TODO: what is in this leaf when `num_vcpu` saturates u8? + let logical_procs: u8 = + self.num_vcpu.get().try_into().map_err(|_| { + SpecializeError::IncompatibleTopology { + leaf: 1, + value: self.num_vcpu.get() as u32, + why: Some( + "Don't know how to set CPUID leaf 1 processor \ + count if there are more than 254 threads!", + ), + } + })?; + ent.ebx &= !0xff0000; + ent.ebx |= u32::from(logical_procs) << 16; } } @@ -163,7 +196,7 @@ impl Specializer { set: &mut CpuidSet, ) -> Result<(), SpecializeError> { assert!(self.do_cache_topo); - let num = self.num_vcpu.unwrap().get(); + let num = self.num_vcpu.get(); for ecx in 0..u32::MAX { match set.get_mut(CpuidIdent::subleaf(0x8000001d, ecx)) { None => break, @@ -200,10 +233,7 @@ impl Specializer { fn fix_cpu_topo(&self, set: &mut CpuidSet) -> Result<(), SpecializeError> { // The number of logical threads available to the guest. - let num_vcpu = self - .num_vcpu - .ok_or(SpecializeError::MissingVcpuCount) - .map(|n| u32::from(n.get()))?; + let num_vcpu = u32::from(self.num_vcpu.get()); // The number of logical processors the guest should see. If we // indicate that SMT is enabled, then vCPUs are presented as pairs @@ -236,19 +266,47 @@ impl Specializer { if !self.cpu_topo_populate.contains(topo) { // We aren't fixing up this leaf, we're just asked to entirely // discard it. - set.remove_leaf(leaf); - - if *topo == TopoKind::Ext1E { - let ext_features = CpuidIdent::leaf(0x8000_0001); - // If the CPUID profile defines leaf 0x8000_0001, Extended - // Processor and Feature Identifier, make sure the bit that - // would indicate leaf Ext1E support is clear. We've just - // discarded the leaf, so there's no meaningful data here. - if let Some(features) = set.get_mut(ext_features) { - // APM volume 3, "CPUID Fn8000_0001_ECX Feature - // Identifiers" - const TOPO_EXTENSIONS_BIT: u32 = 1 << 22; - features.ecx &= !TOPO_EXTENSIONS_BIT; + match topo { + TopoKind::Std4 | TopoKind::StdB => { + // To "clear" the topology indicated by leaves 4 or Bh, + // we'll remove the leaf outright. Proceeding with a + // cleared leaf 4 is probably undesirable, since it's + // likely that the standard CPUID leaf limit will imply + // it's valid even as we've zeroed it, and a guest will + // interpret the zeroes as the intended values. + set.remove_leaf(leaf); + } + TopoKind::Std1F => { + // TODO: handle 0x1f topo info + } + TopoKind::Ext8 => { + // We can't just remove leaf 8000_0008 if we're + // "clearing" the topology: if it's included by the + // extended leaf limit then the address size identifiers + // in EAX are likely needed by guests, and the bits in + // EBX may be interesting. + let leaf_ident = CpuidIdent::leaf(0x8000_0008); + if let Some(features) = set.get_mut(leaf_ident) { + // APM volume 3, "CPUID Fn8000_0008_ECX Size + // Identifiers". Mask of both ApicIdSize and NC. + const TOPO_FIELDS_MASK: u32 = 0x00_00_f0_ff; + features.ecx &= !TOPO_FIELDS_MASK; + } + } + TopoKind::Ext1E => { + set.remove_leaf(leaf); + + let ext_features = CpuidIdent::leaf(0x8000_0001); + // If the CPUID profile defines leaf 0x8000_0001, Extended + // Processor and Feature Identifier, make sure the bit that + // would indicate leaf Ext1E support is clear. We've just + // discarded the leaf, so there's no meaningful data here. + if let Some(features) = set.get_mut(ext_features) { + // APM volume 3, "CPUID Fn8000_0001_ECX Feature + // Identifiers" + const TOPO_EXTENSIONS_BIT: u32 = 1 << 22; + features.ecx &= !TOPO_EXTENSIONS_BIT; + } } } @@ -287,7 +345,7 @@ impl Specializer { if num_vproc >= 0b100_0000 { return Err(SpecializeError::IncompatibleTopology { leaf, - num_vcpu, + value: num_vcpu, why: Some( "Don't know how to set CPUID leaf 4 processor \ count if there are more than 64 processors!", @@ -296,7 +354,7 @@ impl Specializer { } else if num_vproc == 0 { return Err(SpecializeError::IncompatibleTopology { leaf, - num_vcpu, + value: num_vcpu, why: Some( "Cannot specialize CPUID leaf 4 \ for 0 processor VM", @@ -356,7 +414,7 @@ impl Specializer { return Err( SpecializeError::IncompatibleTopology { leaf, - num_vcpu, + value: num_vcpu, why: Some("too many vCPUs"), }, ); @@ -402,7 +460,7 @@ impl Specializer { // least likely to surprise guest OSes. return Err(SpecializeError::IncompatibleTopology { leaf, - num_vcpu, + value: num_vcpu, why: Some("Leaf B.1 would have EAX=0"), }); } @@ -418,7 +476,7 @@ impl Specializer { if num_vcpu >= 256 { return Err(SpecializeError::IncompatibleTopology { leaf, - num_vcpu, + value: num_vcpu, why: Some( "Don't know how to specialize CPUID leaf \ B for more than 256 processors!", @@ -438,12 +496,78 @@ impl Specializer { TopoKind::Std1F => { // TODO: add 0x1f topo info } + TopoKind::Ext8 => { + let ext_features = CpuidIdent::leaf(0x8000_0008); + let ext_8h = set + .get_mut(ext_features) + .expect("leaf to specialize exists"); + ext_8h.ecx &= !0xff; + if num_vcpu >= 256 { + return Err(SpecializeError::IncompatibleTopology { + leaf, + value: num_vcpu, + why: Some( + "Don't know how to specialize CPUID leaf \ + 8000_0008 for more than 256 processors!", + ), + }); + } + ext_8h.ecx |= num_vcpu - 1; + let apic_id_size = num_vcpu.ilog2() + 1; + if apic_id_size > 0b1111 { + return Err(SpecializeError::IncompatibleTopology { + leaf, + value: apic_id_size, + why: Some( + "APIC ID size is larger than leaf Ext8h ECX \ + can express?", + ), + }); + } + ext_8h.ecx |= apic_id_size << 12; + } TopoKind::Ext1E => { let id = self.vcpuid.unwrap_or(0) as u32; let mut ebx = id; if self.has_smt { // bits 15:8 hold the zero-based threads-per-compute-unit ebx |= 0x100; + // bits 7:0 hold the compute-unit-id. a "compute unit" + // is "one or more physical cores", so it is an + // arbitrary choice to pick the upper bits of APIC IDs + // as the compute unit ID. + let compute_unit_id: u8 = + (id >> 1).try_into().map_err(|_| { + SpecializeError::IncompatibleTopology { + leaf: 0x8000_001e, + value: id, + why: Some( + "Need a more coarse compute unit ID \ + for leaf 8000_001e and more than \ + 256 processors!", + ), + } + })?; + ebx |= compute_unit_id as u32; + } else { + // No SMT means the zero-based threads-per-compute-unit is 0. + ebx &= !0x0000ff00; + // bits 7:0 hold the compute-unit-id. Like above, the + // compute unit ID selection is arbitrary (that we know + // of). + let compute_unit_id: u8 = + id.try_into().map_err(|_| { + SpecializeError::IncompatibleTopology { + leaf: 0x8000_001e, + value: id, + why: Some( + "Need a more coarse compute unit ID \ + for leaf 8000_001e and more than \ + 256 processors!", + ), + } + })?; + ebx |= compute_unit_id as u32; } set.insert( CpuidIdent::leaf(leaf), @@ -459,6 +583,18 @@ impl Specializer { edx: 0, }, ); + + let ext_features = CpuidIdent::leaf(0x8000_0001); + // If the CPUID profile defines leaf 0x8000_001E, we should + // also set the bit in Extended Processor and Feature + // Identifier that indicates the extended topology leaves + // are meaningful. + if let Some(features) = set.get_mut(ext_features) { + // APM volume 3, "CPUID Fn8000_0001_ECX Feature + // Identifiers" + const TOPO_EXTENSIONS_BIT: u32 = 1 << 22; + features.ecx |= TOPO_EXTENSIONS_BIT; + } } } } @@ -477,6 +613,8 @@ pub enum TopoKind { StdB = 0xb, /// Leaf 0x1F (Intel) Std1F = 0x1f, + /// LEAF 0x80000008 (AMD) + Ext8 = 0x80000008, /// LEAF 0x8000001E (AMD) Ext1E = 0x8000001e, } @@ -485,10 +623,10 @@ impl TopoKind { /// Return an iterator of the CPU topology information that Propolis /// supports specializing. This is intended as a "reasonable default" for /// CPUID profile specialization. - pub fn supported() -> std::array::IntoIter { - // Topology leaves 1Fh and 8000_001E are partially or entirely TODO, so - // we can't specialize them. - [TopoKind::Std4, TopoKind::StdB].into_iter() + pub fn supported() -> std::array::IntoIter { + // Topology leaf 1Fh is entirely TODO, so we can't specialize it. + [TopoKind::Std4, TopoKind::StdB, TopoKind::Ext8, TopoKind::Ext1E] + .into_iter() } } diff --git a/lib/propolis/src/firmware/acpi/madt.rs b/lib/propolis/src/firmware/acpi/madt.rs index ce0ee608b..cc2133c51 100644 --- a/lib/propolis/src/firmware/acpi/madt.rs +++ b/lib/propolis/src/firmware/acpi/madt.rs @@ -27,7 +27,7 @@ pub struct MadtConfig { pub acpi_variant: AcpiVariant, /// Number of vCPUs in the VM. - pub num_cpus: u8, + pub num_cpus: u16, } /// The MADT/APIC table describes the interrupts for the entire system. @@ -64,7 +64,14 @@ impl Aml for Madt { // Processor Local APIC. // ACPI rev. 6.6 section 5.2.12.2 "Processor Local APIC Structure" - for i in 0..self.config.num_cpus { + let cpus_u8: u8 = + self.config.num_cpus.try_into().unwrap_or_else(|_| { + panic!( + "Don't know how to configure MADT xAPIC/\ + x2APIC tables for >255 CPUs" + ); + }); + for i in 0..cpus_u8 { table.add_structure(madt::ProcessorLocalApic::new( i, i, diff --git a/lib/propolis/src/hw/qemu/fwcfg.rs b/lib/propolis/src/hw/qemu/fwcfg.rs index 316f14b8a..c87aa47e8 100644 --- a/lib/propolis/src/hw/qemu/fwcfg.rs +++ b/lib/propolis/src/hw/qemu/fwcfg.rs @@ -1316,7 +1316,7 @@ pub mod formats { /// Instance configuration that are relevant when building the ACPI tables. pub struct AcpiConfig<'a> { pub acpi_variant: acpi::AcpiVariant, - pub num_cpus: u8, + pub num_cpus: u16, pub pci_window_32: PciWindow, pub pci_window_64: PciWindow, pub dsdt_generators: &'a [&'a dyn acpi::DsdtGenerator], diff --git a/lib/propolis/src/vmm/machine.rs b/lib/propolis/src/vmm/machine.rs index 19a9b3970..1dc477a7e 100644 --- a/lib/propolis/src/vmm/machine.rs +++ b/lib/propolis/src/vmm/machine.rs @@ -178,7 +178,7 @@ impl Machine { pub struct Builder { inner_hdl: Option>, physmap: Option, - max_cpu: u8, + max_cpu: u16, guest_hv_interface: Option>, } impl Builder { @@ -241,8 +241,8 @@ impl Builder { Ok(self) } /// Sets the maximum number of CPUs for the machine. - pub fn max_cpus(mut self, max: u8) -> Result { - if max == 0 || max > MAXCPU as u8 { + pub fn max_cpus(mut self, max: u16) -> Result { + if max == 0 || max > MAXCPU as u16 { Err(Error::new(ErrorKind::InvalidInput, "maxcpu out of range")) } else { self.max_cpu = max; diff --git a/openapi/propolis-server/propolis-server-6.0.0-b5b984.json b/openapi/propolis-server/propolis-server-7.0.0-853fed.json similarity index 96% rename from openapi/propolis-server/propolis-server-6.0.0-b5b984.json rename to openapi/propolis-server/propolis-server-7.0.0-853fed.json index 1bfc621c4..5c11acfdf 100644 --- a/openapi/propolis-server/propolis-server-6.0.0-b5b984.json +++ b/openapi/propolis-server/propolis-server-7.0.0-853fed.json @@ -7,7 +7,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "6.0.0" + "version": "7.0.0" }, "paths": { "/instance": { @@ -485,6 +485,14 @@ } ] }, + "cpu_topology": { + "description": "A *very* simplistic description of this VM's CPU topology.", + "allOf": [ + { + "$ref": "#/components/schemas/CpuTopology" + } + ] + }, "cpuid": { "nullable": true, "description": "The CPUID values to expose to the guest. If `None`, bhyve will derive default values from the host's CPUID values.", @@ -497,7 +505,7 @@ "cpus": { "description": "The number of virtual logical processors attached to this VM.", "type": "integer", - "format": "uint8", + "format": "uint16", "minimum": 0 }, "guest_hv_interface": { @@ -517,6 +525,7 @@ }, "required": [ "chipset", + "cpu_topology", "cpus", "memory_mb" ], @@ -927,6 +936,29 @@ } ] }, + "CpuTopology": { + "description": "A *very* simple topology for for a VM's vCPUs.\n\nTypically this will be configured as `threads_per_core: 2` to indicate SMT, and one socket. The number of vCPUs in each socket is the same, meaning that the number of vCPUs must divide evenly into the number of sockets in the topology.", + "type": "object", + "properties": { + "has_smt": { + "type": "boolean" + }, + "sockets": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "topo_leaves": { + "$ref": "#/components/schemas/TopoLeaves" + } + }, + "required": [ + "has_smt", + "sockets", + "topo_leaves" + ], + "additionalProperties": false + }, "Cpuid": { "description": "A set of CPUID values to expose to a guest.", "type": "object", @@ -2095,6 +2127,31 @@ } ] }, + "TopoLeaves": { + "description": "A collection of CPUID leaves to specialize for a board's `CpuTopology`.\n\nLeaves may be selected here without being present in `Cpuid`, in which case the extra selected topology leaves are ignored. Conversely, `Cpuid` leaves with topology information that are not mentioend here probably indicate a bug (at best, those leaves will probably indicate collection of single-core sockets in the machine.)", + "type": "object", + "properties": { + "ext_1e": { + "type": "boolean" + }, + "ext_8": { + "type": "boolean" + }, + "std_4": { + "type": "boolean" + }, + "std_b": { + "type": "boolean" + } + }, + "required": [ + "ext_1e", + "ext_8", + "std_4", + "std_b" + ], + "additionalProperties": false + }, "UpstairsInfoStatus": { "type": "string", "enum": [ diff --git a/openapi/propolis-server/propolis-server-latest.json b/openapi/propolis-server/propolis-server-latest.json index 114ec385b..d64d9a5ad 120000 --- a/openapi/propolis-server/propolis-server-latest.json +++ b/openapi/propolis-server/propolis-server-latest.json @@ -1 +1 @@ -propolis-server-6.0.0-b5b984.json \ No newline at end of file +propolis-server-7.0.0-853fed.json \ No newline at end of file diff --git a/phd-tests/framework/src/lib.rs b/phd-tests/framework/src/lib.rs index 7c12af978..4f038914c 100644 --- a/phd-tests/framework/src/lib.rs +++ b/phd-tests/framework/src/lib.rs @@ -72,7 +72,7 @@ pub struct Framework { pub(crate) tmp_directory: Utf8PathBuf, pub(crate) log_config: LogConfig, - pub(crate) default_guest_cpus: u8, + pub(crate) default_guest_cpus: u16, pub(crate) default_guest_memory_mib: u64, pub(crate) default_guest_os_artifact: String, pub(crate) default_bootrom_artifact: String, @@ -110,7 +110,7 @@ pub struct FrameworkParameters<'a> { pub artifact_toml: Utf8PathBuf, pub log_config: LogConfig, - pub default_guest_cpus: u8, + pub default_guest_cpus: u16, pub default_guest_memory_mib: u64, pub default_guest_os_artifact: String, pub default_bootrom_artifact: String, diff --git a/phd-tests/framework/src/test_vm/config.rs b/phd-tests/framework/src/test_vm/config.rs index 887f17244..e209b20ae 100644 --- a/phd-tests/framework/src/test_vm/config.rs +++ b/phd-tests/framework/src/test_vm/config.rs @@ -8,10 +8,11 @@ use anyhow::Context; use cpuid_utils::CpuidIdent; use propolis_client::{ instance_spec::{ - Board, BootOrderEntry, BootSettings, Chipset, Component, Cpuid, - CpuidEntry, CpuidVendor, GuestHypervisorInterface, InstanceMetadata, - InstanceSpec, MigrationFailureInjector, NvmeDisk, PciPath, SerialPort, - SerialPortNumber, SpecKey, VirtioDisk, VirtioSocket, + Board, BootOrderEntry, BootSettings, Chipset, Component, CpuTopology, + Cpuid, CpuidEntry, CpuidVendor, GuestHypervisorInterface, + InstanceMetadata, InstanceSpec, MigrationFailureInjector, NvmeDisk, + PciPath, SerialPort, SerialPortNumber, SpecKey, VirtioDisk, + VirtioSocket, }, support::nvme_serial_from_str, }; @@ -62,7 +63,7 @@ struct DiskRequest<'a> { pub struct VmConfig<'dr> { vm_name: String, - cpus: u8, + cpus: u16, memory_mib: u64, cpuid: Option>, bootrom_artifact: String, @@ -71,12 +72,13 @@ pub struct VmConfig<'dr> { migration_failure: Option, guest_hv_interface: Option, vsock: Option, + cpu_topology: CpuTopology, } impl<'dr> VmConfig<'dr> { pub(crate) fn new( vm_name: &str, - cpus: u8, + cpus: u16, memory_mib: u64, bootrom: &str, guest_artifact: &'dr str, @@ -92,6 +94,7 @@ impl<'dr> VmConfig<'dr> { migration_failure: None, guest_hv_interface: None, vsock: None, + cpu_topology: CpuTopology::single_socket_smt(), }; config.boot_disk( @@ -104,7 +107,7 @@ impl<'dr> VmConfig<'dr> { config } - pub fn cpus(&mut self, cpus: u8) -> &mut Self { + pub fn cpus(&mut self, cpus: u16) -> &mut Self { self.cpus = cpus; self } @@ -129,6 +132,11 @@ impl<'dr> VmConfig<'dr> { self } + pub fn cpu_topology(&mut self, cpu_topology: CpuTopology) -> &mut Self { + self.cpu_topology = cpu_topology; + self + } + pub fn guest_hv_interface( &mut self, interface: GuestHypervisorInterface, @@ -241,6 +249,7 @@ impl<'dr> VmConfig<'dr> { migration_failure, guest_hv_interface, vsock, + cpu_topology, } = self; let framework = &ctx.framework; let bootrom_path = framework @@ -322,6 +331,7 @@ impl<'dr> VmConfig<'dr> { .as_ref() .cloned() .unwrap_or_default(), + cpu_topology: *cpu_topology, }, components: Default::default(), smbios: None, diff --git a/phd-tests/runner/src/config.rs b/phd-tests/runner/src/config.rs index 88fe66617..d41d62d59 100644 --- a/phd-tests/runner/src/config.rs +++ b/phd-tests/runner/src/config.rs @@ -159,7 +159,7 @@ pub struct RunOptions { /// The number of CPUs to assign to the guest in tests where the test is /// using the default machine configuration. #[clap(long, value_parser, default_value = "2")] - pub default_guest_cpus: u8, + pub default_guest_cpus: u16, /// The amount of memory, in MiB, to assign to the guest in tests where the /// test is using the default machine configuration. diff --git a/phd-tests/tests/Cargo.toml b/phd-tests/tests/Cargo.toml index 568b24ee2..d0500a413 100644 --- a/phd-tests/tests/Cargo.toml +++ b/phd-tests/tests/Cargo.toml @@ -12,6 +12,7 @@ acpi_tables.workspace = true anyhow.workspace = true backoff.workspace = true byteorder.workspace = true +cfg-if.workspace = true chrono.workspace = true cpuid_utils.workspace = true dropshot.workspace = true diff --git a/phd-tests/tests/src/cpuid.rs b/phd-tests/tests/src/cpuid.rs index 68fe21fbd..b283e1d11 100644 --- a/phd-tests/tests/src/cpuid.rs +++ b/phd-tests/tests/src/cpuid.rs @@ -5,7 +5,9 @@ use cpuid_utils::{CpuidIdent, CpuidSet, CpuidValues}; use phd_framework::{test_vm::MigrationTimeout, TestVm}; use phd_testcase::*; -use propolis_client::instance_spec::{CpuidEntry, InstanceSpecStatus}; +use propolis_client::instance_spec::{ + CpuTopology, CpuidEntry, InstanceSpecStatus, +}; use tracing::info; use uuid::Uuid; @@ -104,9 +106,21 @@ async fn verify_guest_brand_string(vm: &TestVm) -> anyhow::Result<()> { /// Launches a test VM with a synthetic brand string injected into its CPUID /// leaves. +/// +/// Some tests cover specific CPU topologies, for which this helper is +/// unsuitable; instead, use `launch_cpuid_smoke_test_vm_with_topo` and use +/// explicit settings for the code under test. async fn launch_cpuid_smoke_test_vm( ctx: &TestCtx, vm_name: &str, +) -> anyhow::Result { + launch_cpuid_smoke_test_vm_with_topo(ctx, vm_name, None).await +} + +async fn launch_cpuid_smoke_test_vm_with_topo( + ctx: &TestCtx, + vm_name: &str, + guest_topo: Option, ) -> anyhow::Result { let mut host_cpuid = cpuid_utils::host::query_complete( cpuid_utils::host::CpuidSource::BhyveDefault, @@ -130,6 +144,9 @@ async fn launch_cpuid_smoke_test_vm( }) .collect(), ); + if let Some(topo) = guest_topo { + cfg.cpu_topology(topo); + } let mut vm = ctx.spawn_vm(&cfg, None).await?; vm.launch().await?; vm.wait_to_boot().await?; @@ -173,11 +190,11 @@ struct CacheInfo { } impl<'a> LinuxGuestTopo<'a> { - fn cpu_stem(vcpu: u8) -> String { + fn cpu_stem(vcpu: u16) -> String { format!("/sys/devices/system/cpu/cpu{vcpu}/topology") } - fn cache_stem(vcpu: u8, index: u8) -> String { + fn cache_stem(vcpu: u16, index: u8) -> String { format!("/sys/devices/system/cpu/cpu{vcpu}/cache/index{index}") } @@ -209,21 +226,7 @@ impl<'a> LinuxGuestTopo<'a> { this } - async fn vendor_string(&self) -> String { - let command = "cat /proc/cpuinfo | \ - grep vendor_id | \ - head -n 1 | \ - cut -d':' -f 2"; - let out = self - .vm - .run_shell_command(command) - .await - .expect("can grep vendor_id out of cpuinfo"); - - out.trim().to_string() - } - - async fn cpus(&self) -> u8 { + async fn cpus(&self) -> u16 { let spec_get_response = self.vm.get_spec().await.expect("can get the instance's spec back"); let InstanceSpecStatus::Present(spec) = spec_get_response.spec else { @@ -310,6 +313,7 @@ impl<'a> LinuxGuestTopo<'a> { let level_out = self .vm .run_shell_command(&format!("cat {stem}/level 2>/dev/null")) + .ignore_status() .await .expect("can run cat"); @@ -334,75 +338,91 @@ impl<'a> LinuxGuestTopo<'a> { } } -#[phd_testcase] -async fn guest_cpu_topo_test(ctx: &TestCtx) { - let vm = launch_cpuid_smoke_test_vm(ctx, "guest_cpu_topo_test").await?; +async fn check_legacy_topo(guest_topo: &LinuxGuestTopo<'_>) { + // Because the "legacy" (AMD) topology is single-thread sockets, a bunch of + // things that are seemingly bogus. - // The topology-checking is Linux-specific, though it should be appropriate - // for all Linux distributions. For other OSes, just warn and skip for now. - if !vm.guest_os_kind().is_linux() { - phd_skip!(format!( - "guest topo test does not yet have support for {:?}", - vm.guest_os_kind() - )); + // All cores should be in a distinct socket matching their APIC ID. + for (idx, item) in guest_topo.physical_package_ids().await.enumerate() { + let item: usize = item.try_into().unwrap(); + assert_eq!(idx, item); } - // Between the way we set initial APIC IDs and the way Linux numbers logical - // processors, a 4-vCPU VM should report a topology like: - // * core 0: thread 0, thread 1 - // * core 1: thread 2, thread 3 - // - // over in sysfs, the CPU topology files then represent sibling-ness as a - // bitfield. So core 0 has two threads (0, 1) and their bits are set in core - // 0's thread_slibings (0b0000_0011 -> "3"). Core 1 does the same for - // threads 2 and 3, which yields an expected core 1 thread_siblings of - // (0b0000_1100 -> "c"). - let guest_topo = LinuxGuestTopo::new(&vm).await; - - // Except that for the time being we exclude leaf B from specialization (see - // the comment over in `propolis-server/src/lib/initializer.rs`), meaning - // that on AMD there isn't enough topology information for Linux to - // determine anything other than "single-thread single-core many-socket". - // That makes this test Intel-only for the time being. - if guest_topo.vendor_string().await != "GenuineIntel" { - phd_skip!("guest topo test is Intel-only for the moment"); + // Single-thread sockets, so no SMT. + assert!(!guest_topo.has_smt().await); + + // thread_siblings() should only have a single bit, matching the core's ID. + let siblings: Vec<_> = guest_topo.thread_siblings().await.collect(); + for (idx, siblings) in siblings.iter().enumerate() { + let core_char = idx / 2; + let mask = 1 << (idx % 4); + // A truly egregious conversion from single digit integer to `char`, but + // I don't want you to have to read `(0x30 + mask) as char` + let mask_str = mask.to_string(); + assert_eq!(mask_str.len(), 1); + let expected_char = mask_str.chars().next().unwrap(); + assert!(siblings.chars().enumerate().all(|(i, ch)| { + if i != core_char { + ch == '0' + } else { + ch == expected_char + } + })); } +} +// Between the way we set initial APIC IDs and the way Linux numbers logical +// processors, a 4-vCPU VM should report a topology like: +// * core 0: thread 0, thread 1 +// * core 1: thread 2, thread 3 +// +// over in sysfs, the CPU topology files then represent sibling-ness as a +// bitfield. So core 0 has two threads (0, 1) and their bits are set in core +// 0's thread_slibings (0b0000_0011 -> "3"). Core 1 does the same for +// threads 2 and 3, which yields an expected core 1 thread_siblings of +// (0b0000_1100 -> "c"). +async fn check_smt_topo(guest_topo: &LinuxGuestTopo<'_>) { // All cores should be in socket 0 assert!(guest_topo.physical_package_ids().await.all(|item| item == 0)); - let has_smt = guest_topo.has_smt().await; - - // The thread_siblings checks only make sense when SMT is enabled. - if has_smt { - // We currently number CPUs such that Linux numbers them as successive - // pairs of thread twins. - let siblings: Vec<_> = guest_topo.thread_siblings().await.collect(); - for (idx, pair) in siblings.chunks(2).enumerate() { - assert!(pair.len() == 2, "expected even number of CPUs"); - let lower = &pair[0]; - let upper = &pair[1]; - - // Each pair of siblings should see that they have the same siblings - assert_eq!(lower, upper); - // Each hex digit represents 4 CPUs. With chunks(2), idx is the pair - // number: idx=0 is cpus 0-1, idx=1 is cpus 2-3, etc. - let sibling_idx = idx / 2; - let sibling_char = if idx % 2 == 0 { '3' } else { 'c' }; - assert!(lower.chars().enumerate().all(|(i, ch)| { - if i != sibling_idx { - ch == '0' - } else { - ch == sibling_char - } - })); - } + assert!(guest_topo.has_smt().await); + + // We currently number CPUs such that Linux numbers them as successive + // pairs of thread twins. + let siblings: Vec<_> = guest_topo.thread_siblings().await.collect(); + for (idx, pair) in siblings.chunks(2).enumerate() { + assert!(pair.len() == 2, "expected even number of CPUs"); + let lower = &pair[0]; + let upper = &pair[1]; + + // Each pair of siblings should see that they have the same siblings + assert_eq!(lower, upper); + // Each hex digit represents 4 CPUs. With chunks(2), idx is the pair + // number: idx=0 is cpus 0-1, idx=1 is cpus 2-3, etc. + let sibling_idx = idx / 2; + let sibling_char = if idx % 2 == 0 { '3' } else { 'c' }; + assert!(lower.chars().enumerate().all(|(i, ch)| { + if i != sibling_idx { + ch == '0' + } else { + ch == sibling_char + } + })); } +} - // Check cache topology. With SMT, L1/L2 should be shared by SMT siblings - // while L3 is shared across all vCPUs. +// Check cache topology from the perspective of core 0. +// +// We should see L1i, L1d, L2, and L3 caches, where L1 and L2 caches are shared +// iff SMT is enabled. L3's sharing depends on the CPU topology, so what we +// should expect is really only known by the caller. +async fn check_cache_topo( + guest_topo: &LinuxGuestTopo<'_>, + expected_l1_caches: usize, + l3_sharing: &str, +) { let caches = guest_topo.cache_info().await; - let num_cpus = guest_topo.cpus().await; + let has_smt = guest_topo.has_smt().await; for cache in &caches { match cache.level { @@ -411,8 +431,7 @@ async fn guest_cpu_topo_test(ctx: &TestCtx) { assert_eq!(cache.shared_cpu_list, expected); } 3 => { - let expected = format!("0-{}", num_cpus - 1); - assert_eq!(cache.shared_cpu_list, expected); + assert_eq!(cache.shared_cpu_list, l3_sharing); } other => { panic!("unexpected cache level {other}"); @@ -422,6 +441,166 @@ async fn guest_cpu_topo_test(ctx: &TestCtx) { let l1_count = caches.iter().filter(|c| c.level == 1).count(); let l2_count = caches.iter().filter(|c| c.level == 2).count(); - assert!(l1_count >= 2, "expected at least L1d and L1i caches"); + + if l1_count != expected_l1_caches { + match expected_l1_caches { + 1 => { + panic!("expected one L1 cache, found {}", l1_count); + } + 2 => { + panic!("expected L1d and L1i caches, found {}", l1_count); + } + other => { + panic!("expected {} L1 caches, found {}", other, l1_count); + } + }; + } assert!(l2_count >= 1, "expected at least one L2 cache"); } + +/// This intentionally-reductive list of CPU vendors is only because Propolis is +/// primarily tested on AMD x86-64 processors, is expected to reasonably work on +/// Intel x86-64 processors, and expects no particular suppport elsewhere. +/// (yet?) +enum CpuVendor { + Intel, + Amd, +} + +fn host_cpu_vendor() -> CpuVendor { + let mut vendor_bytes = [0u8; 12]; + cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + let result = core::arch::x86_64::__cpuid_count(0, 0); + vendor_bytes[0..4].copy_from_slice(&result.ebx.to_le_bytes()); + vendor_bytes[4..8].copy_from_slice(&result.edx.to_le_bytes()); + vendor_bytes[8..12].copy_from_slice(&result.ecx.to_le_bytes()); + } else { + // You're running PHD on something other than x86-64? + // Have you ported Propolis to ARM? Cool! + panic!("host CPUID queries only work on x86-64 hosts"); + } + } + match &vendor_bytes { + b"GenuineIntel" => CpuVendor::Intel, + b"AuthenticAMD" => CpuVendor::Amd, + other => { + let brand_string = String::from_utf8_lossy(other.as_slice()); + panic!("unknown CPUID brand string: {}", brand_string); + } + } +} + +#[phd_testcase] +async fn guest_cpu_legacy_topo_test(ctx: &TestCtx) { + // As of the introduction of explicit CPU topology configuration, the extant + // Intel behavior was to provide a single socket, while the extant AMD + // behavior was, accidentally, to provide a single-thread-per-socket + // topology. + // + // Now, the test harness picks a single-socket default, and + // topology-sensitive tests pick expected VM topologies specifically. This + // test is "the prior Propolis topology is unchanged". The only reason we'd + // care to preserve the single-thread-per-socket half-specialized topology + // is for use in the product's "initial" virtual platform with several other + // decisions which would break guests to change. + let legacy_topo = match host_cpu_vendor() { + CpuVendor::Intel => CpuTopology::single_socket_smt(), + CpuVendor::Amd => CpuTopology::legacy_single_thread_sockets(2), + }; + + let vm = launch_cpuid_smoke_test_vm_with_topo( + ctx, + "guest_cpu_legacy_topo_test", + Some(legacy_topo), + ) + .await?; + + // The topology-checking is Linux-specific, though it should be appropriate + // for all Linux distributions. For other OSes, just warn and skip for now. + if !vm.guest_os_kind().is_linux() { + phd_skip!(format!( + "guest topo test does not yet have support for {:?}", + vm.guest_os_kind() + )); + } + + let guest_topo = LinuxGuestTopo::new(&vm).await; + + let expected_l3_share = match host_cpu_vendor() { + CpuVendor::Intel => { + let num_cpus = guest_topo.cpus().await; + format!("0-{}", num_cpus - 1) + } + CpuVendor::Amd => "0".to_string(), + }; + + match host_cpu_vendor() { + CpuVendor::Intel => check_smt_topo(&guest_topo).await, + CpuVendor::Amd => check_legacy_topo(&guest_topo).await, + }; + + check_cache_topo(&guest_topo, 2, &expected_l3_share).await; +} + +#[phd_testcase] +async fn guest_cpu_single_socket_smt(ctx: &TestCtx) { + // Now and in the future we can explicitly request a CPU topology via + // propolis-server's API, and control which leaves reflect that topology. In + // all cases if we set a single-socket topology, that's what the guest + // should show. + let test_topo = CpuTopology::single_socket_smt(); + + let vm = launch_cpuid_smoke_test_vm_with_topo( + ctx, + "guest_cpu_single_socket_smt", + Some(test_topo), + ) + .await?; + + // The topology-checking is Linux-specific, though it should be appropriate + // for all Linux distributions. For other OSes, just warn and skip for now. + if !vm.guest_os_kind().is_linux() { + phd_skip!(format!( + "guest topo test does not yet have support for {:?}", + vm.guest_os_kind() + )); + } + + let guest_topo = LinuxGuestTopo::new(&vm).await; + + let num_cpus = guest_topo.cpus().await; + let expected_l3_share = format!("0-{}", num_cpus - 1); + + let expected_l1_caches = match host_cpu_vendor() { + CpuVendor::Intel => 2, + CpuVendor::Amd => { + // TODO: PHD does not yet get the full CPUID profile leaves that + // Omicron would send a real Propolis in the product, which means + // that the CPU topology we test against here is Propolis' + // specialization of bhyve defaults. Since we enable + // TopologyExtensions the guest may reasonably consider leaf + // 8000_001D, but legacy_emulate_cpuid will present a three-cache + // layout with an L1, an L2, and an L3. + // + // In the product we present a Zen-like L1i, L1d, L2, and L3, so we + // *should* expect two L1 caches here in a product-like VM. When we + // extract virtual platform details from Omicron and can use them + // here, PHD VMs will have two L1s as they should and we'll update + // this to expect two L1 caches in either vendor's case. + // + // XXX: It's pretty weird we'd look for two L1 caches above, but now + // with the default bhyve 8000_001D setup we're back to one, right? + // The two-L1 notion comes from leaf 8000_0005, which it turns out + // bhyve does not mask, so there are non-zero-size L1i and L1d. When + // 8000_001D is available it takes precedence for the layout, so + // bhyve's default topology applies instead. This is probably a bug + // to fix in bhyve's legacy_emulate_cpuid to make these consistent. + 1 + } + }; + + check_smt_topo(&guest_topo).await; + + check_cache_topo(&guest_topo, expected_l1_caches, &expected_l3_share).await; +} diff --git a/scripts/cpuid-queries.d b/scripts/cpuid-queries.d index 9c88dcb1a..93b792499 100755 --- a/scripts/cpuid-queries.d +++ b/scripts/cpuid-queries.d @@ -33,7 +33,9 @@ BEGIN { target_vm = $1; } -fbt::vcpu_emulate_cpuid:entry / arg0 == target_vm / { +fbt::vcpu_emulate_cpuid:entry / + arg0 == target_vm +/ { self->rax = (uint64_t*)arg2; self->rbx = (uint64_t*)arg3; self->rcx = (uint64_t*)arg4;