From 33cf2d44d66f295c6f636c9a622b0504eeee7acd Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:49:04 -0800 Subject: [PATCH] Expose one CPU feature set to MSHV and WHP guests Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> --- CHANGELOG.md | 2 + docs/README.md | 2 + docs/guest-cpu.md | 35 +++++ docs/msr.md | 64 ++------ docs/snapshot-versioning.md | 4 + .../src/hypervisor/virtual_machine/mod.rs | 4 + .../hypervisor/virtual_machine/mshv/x86_64.rs | 20 ++- .../virtual_machine/partition_features.rs | 112 ++++++++++++++ .../hypervisor/virtual_machine/whp/x86_64.rs | 146 ++++++++++++++---- .../src/sandbox/initialized.rs | 142 +++++++++++++++++ .../src/sandbox/snapshot/file/media_types.rs | 2 +- .../src/sandbox/snapshot/file_tests.rs | 4 +- .../src/sandbox/snapshot/tripwires.rs | 2 +- .../tests/snapshot_goldens/goldens_version.rs | 2 +- src/tests/rust_guests/simpleguest/src/main.rs | 13 ++ 15 files changed, 460 insertions(+), 94 deletions(-) create mode 100644 docs/guest-cpu.md create mode 100644 src/hyperlight_host/src/hypervisor/virtual_machine/partition_features.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e4c94cada..98b34538d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). page size, and log level. * **Breaking:** Encode function calls with serde/postcard instead of FlatBuffers. * Stop resetting guest FPU state between guest calls to improve performance. +* **Breaking:** Expose one processor feature set to MSHV and WHP guests. Leave out speculation control, TSX, CET, AMX, and other stateful features Hyperlight guests do not use. Snapshots use ABI 12, so existing snapshots must be regenerated. +* Run WHP guests in their own speculation domain. ### Fixed * Fix surrogate process creation for Windows paths containing Unicode or spaces. diff --git a/docs/README.md b/docs/README.md index c967e281e..78223fefc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -44,3 +44,5 @@ This project is composed internally of several components, depicted in the below * [Hyperlight Surrogate Development Notes](./hyperlight-surrogate-development-notes.md) * [Debugging Hyperlight](./debugging-hyperlight.md) * [Signal Handling in Hyperlight](./signal-handlers-development-notes.md) +* [MSR state across restore](./msr.md) +* [Guest CPU features](./guest-cpu.md) diff --git a/docs/guest-cpu.md b/docs/guest-cpu.md new file mode 100644 index 000000000..afa5e93bd --- /dev/null +++ b/docs/guest-cpu.md @@ -0,0 +1,35 @@ +# Guest CPU features + +Hyperlight exposes only the CPU features its guests need. Each extra feature +adds guest state that restore must reset. + +## MSHV and WHP + +MSHV and WHP guests see one set of CPU features, limited by the host. Both +backends pass the list in `partition_features.rs` to Hyper-V at partition +creation. + +The list holds only features of the 21H2 compatibility version. The MSHV +driver creates every partition at that version. Hyper-V hides newer features +from such partitions. + +The list keeps instruction-set features such as SSE4.2, BMI2, and AES. They +hold no state. It also keeps AVX through AVX-512. Restore resets their state +with the rest of the XSAVE area. It leaves out: + +| Features | Reason | +| --- | --- | +| Speculation control: IBRS, STIBP, IBPB, SSBD, PSFD, BHI_DIS, VIRT_SPEC_CTRL | Guest `SPEC_CTRL` state. A guest is one trust domain. | +| TSX | `TSX_CTRL` state. | +| CET | Shadow-stack state and CET MSRs. | +| `TSC_ADJUST` | A write to `TSC` or `TSC_ADJUST` shifts both registers equally. Restoring both would undo the TSC rewind. | +| `TSC_DEADLINE`, WAITPKG, MPERF/APERF, RDPRU | Timer, wait, and counter state. RDPRU reads MPERF/APERF. | +| MPX, XFD, AMX | `BNDCFGS` and `XFD` MSR state. AMX needs XFD. | +| Supervisor XSAVE state | Used only by CET and PASID. | +| Hyper-V enlightenments | Guests do not use them. | + +## KVM + +KVM guests get the CPUID that KVM supports, minus VMX, SVM, x2APIC, and CET. +KVM's MSR filter denies every MSR the sandbox does not declare. See +[MSR state across restore](./msr.md). diff --git a/docs/msr.md b/docs/msr.md index 689a738d7..0cc048eb8 100644 --- a/docs/msr.md +++ b/docs/msr.md @@ -37,7 +37,7 @@ The backend owns discovery because register mappings and capabilities differ. Sorting, deduplication, baseline capture, snapshot validation, and the fallback to the destination baseline are shared in `MsrResetState`. -Every reset entry must represent guest-writable retained state the host can read +Every reset entry must represent guest-writable state the host can read and write. On the Hyper-V backends the candidate table is derived from the Hyper-V source and must be audited when that source, register mappings, or feature exposure changes. VM creation probes each entry with a host read and a @@ -125,49 +125,27 @@ poisons the sandbox. ## Hyper-V backends MSHV and WHP cannot filter guest MSR access, so restore scrubs a broad set of -retained MSRs to the destination baseline before writing the saved core and -declared-MSR values. The scrub set is built from: +MSRs that hold guest state to the destination baseline before writing the saved +core and declared-MSR values. The scrub set is built from: -* Retained-state candidates the backend maps and can read, including state - reachable only through the VP register API such as active SSP. +* Candidates the backend maps and can read, including state reachable only + through the VP register API such as active SSP. * MTRRs required by the virtual CPU's `MTRRCAP`. The candidate table is not an intercept list. Hyper-V also intercepts read-only, -command, and host-derived MSRs that retain no guest-controlled value. An entry +command, and host-derived MSRs that keep no guest-controlled value. An entry belongs in the table only when guest execution can leave state that affects -later execution. +later execution. The table ignores the host's MSR index list. That list does +not show which MSRs keep guest state. + +Guest CPU features limit which MSRs MSHV and WHP guests can reach. See +[Guest CPU features](./guest-cpu.md). A future partition-scrub hypercall will reset all MSRs directly and replace the candidate table. WHP has kernel support. MSHV support is in progress. The saved set stays the core plus the declared MSRs across that migration, so restore behavior does not change. -### MSHV - -MSHV enables the processor features supported by the host unless a partition -feature mask disables them. Its guest-visible MSR surface therefore varies by -host CPU. - -MSHV maps `IA32_XSS` through `MSR_IA32_REGISTER_U_XSS`. It maps `IA32_MPERF` -and `IA32_APERF` through the per-VP `MCount` and `ACount` registers. TSX, -WAITPKG, CET, XFD, MPX, and deadline-timer state enter the reset set when the -host exposes and maps them. - -The host's enumerated MSR index list does not identify retained state, so it -does not define the reset set. - -On capable Intel hosts MSHV can expose ENQCMD and PASID. PASID is a supervisor -XSAVE component, so XSAVE restore owns it. - -### WHP - -WHP's default feature banks disable speculation control, experimental -`DEBUGCTL` bits, and performance monitoring. Its supported feature mask omits -ENQCMD and defines no FRED feature, so WHP does not expose PASID or FRED. - -WHP maps supported MSRs to `WHV_REGISTER_NAME` values. The same mapping is used -for snapshot reads and restore writes. - ### MTRRs MSHV and WHP read `IA32_MTRRCAP` during VM creation. The reset set includes @@ -178,17 +156,12 @@ clear. VM creation fails if `VCNT` exceeds the supported maximum of 16 or required MTRRs cannot be read. -### TSC - -Restore sets the guest TSC back to its snapshot value. MSHV and WHP partitions -disable `TscAdjustSupport`. That feature makes a write to `TSC` or `TSC_ADJUST` -move the other by the same amount, so restoring both would undo the rewind. - -## Retained-state inventory +## Guest-state inventory -These MSRs are reset when supported by the selected backend and host. +These MSRs hold guest-writable state. A backend resets each one the host +exposes. MSHV and WHP guests lack the features behind many of them. -| MSR (index) | Retained state | +| MSR (index) | State | | --- | --- | | SYSENTER CS, ESP, EIP (`0x174`-`0x176`) | System-call entry state. | | STAR, LSTAR, CSTAR, SFMASK (`0xC000_0081`-`0xC000_0084`) | System-call target state. | @@ -239,12 +212,5 @@ CPU. It is a regression tool, not a complete inventory of vendor MSRs. MSHV gains kernel support. WHP already has it. * Exercise MSHV and WHP on more CPU models. Their reachable MSR surfaces depend on host features. -* Disable optional stateful features the guest does not need, so the guest - cannot retain that state. This shrinks the reset set toward the always-present - core (SYSENTER, syscall targets, KERNEL_GS_BASE, PAT, DEBUGCTL, TSC, MTRRs), - which every host can read. The reset set then becomes fixed, so the discovery - probe that drops host-unsupported entries becomes unnecessary, and the audit - surface shrinks. Do not add a minimum feature set unless the goal is to bar - some host CPUs from running Hyperlight. * Extend the inventory when Hyperlight enables new CPU features such as perfmon, FRED, or nested virtualization. \ No newline at end of file diff --git a/docs/snapshot-versioning.md b/docs/snapshot-versioning.md index cca22ebc2..14e95c70f 100644 --- a/docs/snapshot-versioning.md +++ b/docs/snapshot-versioning.md @@ -59,6 +59,10 @@ host page size, and log level. Earlier snapshots must be regenerated. ABI 11 encodes control data with postcard and carries its length in a 16-byte message header. Earlier snapshots must be regenerated. +ABI 12 runs MSHV and WHP guests with the CPU features in +`partition_features.rs`. Earlier snapshots can hold MSRs of features those +guests lack and must be regenerated. + The `OCI_LAYOUT_VERSION` constant is pinned by the OCI image-layout spec at `1.0.0`. diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs index 47c7db573..514cb9d24 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs @@ -38,6 +38,10 @@ pub(crate) mod whp; #[cfg(feature = "hw-interrupts")] pub(crate) mod x86_64; +/// Processor features shared by MSHV and WHP partitions +#[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))] +pub(crate) mod partition_features; + static AVAILABLE_HYPERVISOR: OnceLock> = OnceLock::new(); /// Returns which type of hypervisor is available, if any diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs index 1407fceaf..3c5abb38a 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs @@ -13,8 +13,9 @@ use mshv_bindings::LapicState; #[cfg(gdb)] use mshv_bindings::{DebugRegisters, hv_message_type_HVMSG_X64_EXCEPTION_INTERCEPT}; use mshv_bindings::{ - FloatingPointUnit, HV_X64_REGISTER_CLASS_IP, SpecialRegisters, StandardRegisters, XSave, Xcrs, - hv_message_type, hv_message_type_HVMSG_GPA_INTERCEPT, hv_message_type_HVMSG_UNMAPPED_GPA, + FloatingPointUnit, HV_X64_REGISTER_CLASS_IP, MSHV_PT_BIT_CPU_AND_XSAVE_FEATURES, + SpecialRegisters, StandardRegisters, XSave, Xcrs, hv_message_type, + hv_message_type_HVMSG_GPA_INTERCEPT, hv_message_type_HVMSG_UNMAPPED_GPA, hv_message_type_HVMSG_X64_HALT, hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT, hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES, hv_partition_synthetic_processor_features, hv_register_assoc, @@ -44,6 +45,7 @@ use crate::hypervisor::regs::{ }; #[cfg(test)] use crate::hypervisor::virtual_machine::XSAVE_BUFFER_SIZE; +use crate::hypervisor::virtual_machine::partition_features::{PROCESSOR_FEATURES, XSAVE_FEATURES}; #[cfg(feature = "hw-interrupts")] use crate::hypervisor::virtual_machine::x86_64::hw_interrupts::TimerThread; use crate::hypervisor::virtual_machine::{ @@ -171,14 +173,20 @@ impl MshvVm { pub(crate) fn new() -> std::result::Result { let mshv = MSHV.as_ref().map_err(|e| e.clone())?; + // The kernel takes masks of disabled features. #[allow(unused_mut)] - let mut pr: mshv_create_partition_v2 = Default::default(); - // The default has no partition flags. Hardware interrupts add only the - // LAPIC required by request_virtual_interrupt. + let mut pr = mshv_create_partition_v2 { + pt_flags: 1u64 << MSHV_PT_BIT_CPU_AND_XSAVE_FEATURES, + pt_num_cpu_fbanks: PROCESSOR_FEATURES.len() as u16, + pt_cpu_fbanks: PROCESSOR_FEATURES.map(|bank| !bank), + pt_disabled_xsave: !XSAVE_FEATURES, + ..Default::default() + }; + // Hardware interrupts add the LAPIC required by request_virtual_interrupt. #[cfg(feature = "hw-interrupts")] { use mshv_bindings::MSHV_PT_BIT_LAPIC; - pr.pt_flags = 1u64 << MSHV_PT_BIT_LAPIC; + pr.pt_flags |= 1u64 << MSHV_PT_BIT_LAPIC; } // It's important to use create_vm_with_args() (not create_vm()), // because create_vm() sets up a SynIC partition by default. diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/partition_features.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/partition_features.rs new file mode 100644 index 000000000..fedb38ce9 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/partition_features.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2025 The Hyperlight Authors. + +//! Processor features that MSHV and WHP guests see. See `docs/guest-cpu.md`. +//! +//! Bit positions follow `hv_partition_processor_features` and +//! `hv_partition_processor_xsave_features` in Linux `include/hyperv/hvhdk.h`. They match +//! `WHV_PROCESSOR_FEATURES_BANKS` and `WHV_PROCESSOR_XSAVE_FEATURES`. Every bit is part of the +//! 21H2 compatibility version and of the mask WHP accepts. + +const fn bits(list: &[u32]) -> u64 { + let mut mask = 0; + let mut i = 0; + while i < list.len() { + mask |= 1 << list[i]; + i += 1; + } + mask +} + +const BANK0: &[u32] = &[ + 0, // SSE3 + 1, // LAHF/SAHF + 2, // SSSE3 + 3, // SSE4.1 + 4, // SSE4.2 + 5, // SSE4A + 6, // XOP + 7, // POPCNT + 8, // CMPXCHG16B + 9, // ALTMOVCR8 + 10, // LZCNT + 11, // MisalignSSE + 12, // MMX extensions + 13, // 3DNow! + 14, // Extended 3DNow! + 15, // 1 GB pages + 16, // AES + 17, // PCLMULQDQ + 18, // PCID + 19, // FMA4 + 20, // F16C + 21, // RDRAND + 22, // RDFSBASE/WRFSBASE + 23, // SMEP + 24, // Enhanced fast strings + 25, // BMI1 + 26, // BMI2 + 29, // MOVBE + 30, // NPIEP1 + 31, // Deprecated x87 FPU CS/DS + 32, // RDSEED + 33, // ADX + 34, // PREFETCHW + 35, // SMAP + 38, // RDTSCP + 39, // CLFLUSHOPT + 43, // INVPCID + 47, // Unrestricted guest + 49, // Fast short REP MOV + 51, // RDCL_NO + 54, // SSB_NO + 55, // RSBA_NO + 57, // RDPID + 58, // UMIP + 59, // MDS_NO + 60, // MD_CLEAR + 61, // TAA_NO +]; + +const BANK1: &[u32] = &[ + 1, // Invariant TSC + 5, // MBEC + 10, // VMX exception inject + 28, // FB_CLEAR + 29, // BTC_NO + 48, // RFDS_NO + 49, // RFDS_CLEAR +]; + +const XSAVE: &[u32] = &[ + 0, // XSAVE + 1, // XSAVEOPT + 2, // AVX + 3, // AVX2 + 4, // FMA + 6, // AVX512F + 7, // AVX512DQ + 8, // AVX512CD + 9, // AVX512BW + 10, // AVX512VL + 11, // Compacted XSAVE + 14, // AVX512_BITALG + 15, // AVX512_IFMA + 16, // AVX512_VBMI + 17, // AVX512_VBMI2 + 18, // AVX512_VNNI + 19, // GFNI + 20, // VAES + 21, // AVX512_VPOPCNTDQ + 22, // VPCLMULQDQ + 23, // AVX512_BF16 + 24, // AVX512_VP2INTERSECT + 25, // AVX512_FP16 + 30, // AVX_VNNI +]; + +/// Enabled processor feature banks 0 and 1. +pub(crate) const PROCESSOR_FEATURES: [u64; 2] = [bits(BANK0), bits(BANK1)]; + +/// Enabled processor XSAVE features. +pub(crate) const XSAVE_FEATURES: u64 = bits(XSAVE); diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/whp/x86_64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/whp/x86_64.rs index 9c0d27247..fa6f15933 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/whp/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/whp/x86_64.rs @@ -32,6 +32,7 @@ use crate::hypervisor::surrogate_process::SurrogateProcess; use crate::hypervisor::surrogate_process_manager::{ get_surrogate_process_manager, surrogates_disabled, }; +use crate::hypervisor::virtual_machine::partition_features::{PROCESSOR_FEATURES, XSAVE_FEATURES}; #[cfg(feature = "hw-interrupts")] use crate::hypervisor::virtual_machine::x86_64::hw_interrupts::{self, TimerThread}; use crate::hypervisor::virtual_machine::{ @@ -64,6 +65,24 @@ pub(crate) fn is_hypervisor_present() -> bool { } } +/// `value` holds the union field that matches `code`. +fn set_partition_property( + partition: WHV_PARTITION_HANDLE, + code: WHV_PARTITION_PROPERTY_CODE, + value: WHV_PARTITION_PROPERTY, +) -> Result<(), CreateVmError> { + // SAFETY: `value` is a readable buffer of the size passed. + unsafe { + WHvSetPartitionProperty( + partition, + code, + &value as *const _ as *const c_void, + std::mem::size_of_val(&value) as u32, + ) + } + .map_err(|e| CreateVmError::SetPartitionProperty(e.into())) +} + /// Maps an MSR index to the WHP register used for host reset. fn msr_to_whv_register_name(index: u32) -> Option { Some(match index { @@ -278,15 +297,15 @@ impl WhpVm { Self::check_lapic_emulation_support()?; let p = WHvCreatePartition().map_err(|e| CreateVmError::CreateVmFd(e.into()))?; - WHvSetPartitionProperty( + set_partition_property( p, WHvPartitionPropertyCodeProcessorCount, - &NUM_CPU as *const _ as *const _, - std::mem::size_of_val(&NUM_CPU) as _, - ) - .map_err(|e| CreateVmError::SetPartitionProperty(e.into()))?; + WHV_PARTITION_PROPERTY { + ProcessorCount: NUM_CPU, + }, + )?; - Self::disable_tsc_adjust(p)?; + Self::configure_partition(p)?; #[cfg(feature = "hw-interrupts")] Self::enable_lapic_emulation(p)?; @@ -322,37 +341,77 @@ impl WhpVm { }) } - /// `TscAdjustSupport` makes a write to `TSC` or `TSC_ADJUST` move the other - /// by the same amount, so restoring both would undo the TSC rewind. - fn disable_tsc_adjust(partition: WHV_PARTITION_HANDLE) -> Result<(), CreateVmError> { - // Bit 18 of `WHV_PROCESSOR_FEATURES1`, the second bank. - const TSC_ADJUST_SUPPORT: u64 = 1 << 18; - + /// Limits the guest to the processor features in `partition_features` and gives it its own + /// speculation domain. The hypervisor then flushes branch predictors when the guest exits. + fn configure_partition(partition: WHV_PARTITION_HANDLE) -> Result<(), CreateVmError> { let mut banks = WHV_PROCESSOR_FEATURES_BANKS { - BanksCount: 2, + BanksCount: PROCESSOR_FEATURES.len() as u32, ..Default::default() }; - let size = std::mem::size_of_val(&banks) as u32; - // SAFETY: `banks` is a buffer of `size` bytes, and every view of its - // union is plain `u64` data. + // SAFETY: `banks` is a writable buffer of the size passed, and every view of its union is + // plain `u64` data. unsafe { WHvGetPartitionProperty( partition, WHvPartitionPropertyCodeProcessorFeaturesBanks, &mut banks as *mut _ as *mut c_void, - size, + std::mem::size_of_val(&banks) as u32, None, ) .map_err(|e| CreateVmError::GetPartitionProperty(e.into()))?; - banks.Anonymous.AsUINT64[1] &= !TSC_ADJUST_SUPPORT; - WHvSetPartitionProperty( + for (bank, enabled) in banks.Anonymous.AsUINT64.iter_mut().zip(PROCESSOR_FEATURES) { + *bank &= enabled; + } + } + set_partition_property( + partition, + WHvPartitionPropertyCodeProcessorFeaturesBanks, + WHV_PARTITION_PROPERTY { + ProcessorFeaturesBanks: banks, + }, + )?; + + let mut xsave = WHV_PROCESSOR_XSAVE_FEATURES::default(); + // SAFETY: `xsave` is a writable buffer of the size passed, and every view of its union is + // plain `u64` data. + unsafe { + WHvGetPartitionProperty( partition, - WHvPartitionPropertyCodeProcessorFeaturesBanks, - &banks as *const _ as *const c_void, - size, + WHvPartitionPropertyCodeProcessorXsaveFeatures, + &mut xsave as *mut _ as *mut c_void, + std::mem::size_of_val(&xsave) as u32, + None, ) - .map_err(|e| CreateVmError::SetPartitionProperty(e.into())) + .map_err(|e| CreateVmError::GetPartitionProperty(e.into()))?; + xsave.AsUINT64 &= XSAVE_FEATURES; } + set_partition_property( + partition, + WHvPartitionPropertyCodeProcessorXsaveFeatures, + WHV_PARTITION_PROPERTY { + ProcessorXsaveFeatures: xsave, + }, + )?; + + // The guest uses no Hyper-V enlightenments. + set_partition_property( + partition, + WHvPartitionPropertyCodeSyntheticProcessorFeaturesBanks, + WHV_PARTITION_PROPERTY { + SyntheticProcessorFeaturesBanks: WHV_SYNTHETIC_PROCESSOR_FEATURES_BANKS { + BanksCount: 1, + ..Default::default() + }, + }, + )?; + + set_partition_property( + partition, + WHvPartitionPropertyCodeSeparateSecurityDomain, + WHV_PARTITION_PROPERTY { + SeparateSecurityDomain: windows::core::BOOL(1), + }, + ) } fn get_registers( @@ -1301,17 +1360,13 @@ impl WhpVm { /// Enable LAPIC emulation on the given partition. fn enable_lapic_emulation(partition: WHV_PARTITION_HANDLE) -> Result<(), CreateVmError> { - let apic_mode = WHvX64LocalApicEmulationModeXApic; - unsafe { - WHvSetPartitionProperty( - partition, - WHvPartitionPropertyCodeLocalApicEmulationMode, - &apic_mode as *const _ as *const _, - std::mem::size_of_val(&apic_mode) as _, - ) - .map_err(|e| CreateVmError::SetPartitionProperty(e.into()))?; - } - Ok(()) + set_partition_property( + partition, + WHvPartitionPropertyCodeLocalApicEmulationMode, + WHV_PARTITION_PROPERTY { + LocalApicEmulationMode: WHvX64LocalApicEmulationModeXApic, + }, + ) } /// Initialize the LAPIC via the bulk interrupt-controller state API. @@ -1488,6 +1543,29 @@ unsafe fn try_load_whv_map_gpa_range2() -> windows_result::Result [u32; 4] { + let bytes: Vec = sbox.call("Cpuid", (leaf, subleaf)).unwrap(); + std::array::from_fn(|i| { + u32::from_le_bytes(bytes[i * 4..i * 4 + 4].try_into().unwrap()) + }) + }; + + // (leaf, subleaf, register, bit, feature) + let hidden = [ + (1, 0, ECX, 24, "TSC_DEADLINE"), + (6, 0, ECX, 0, "APERF/MPERF"), + (7, 0, EBX, 1, "TSC_ADJUST"), + (7, 0, EBX, 4, "HLE"), + (7, 0, EBX, 11, "RTM"), + (7, 0, EBX, 14, "MPX"), + (7, 0, ECX, 5, "WAITPKG"), + (7, 0, ECX, 7, "CET_SS"), + (7, 0, EDX, 20, "CET_IBT"), + (7, 0, EDX, 22, "AMX_BF16"), + (7, 0, EDX, 24, "AMX_TILE"), + (7, 0, EDX, 25, "AMX_INT8"), + (7, 0, EDX, 26, "IBRS/IBPB"), + (7, 0, EDX, 27, "STIBP"), + (7, 0, EDX, 28, "L1D_FLUSH"), + (7, 0, EDX, 31, "SSBD"), + (7, 2, EDX, 0, "PSFD"), + (7, 2, EDX, 4, "BHI_CTRL"), + (0xD, 1, EAX, 4, "XFD"), + (0x8000_0008, 0, EBX, 4, "RDPRU"), + (0x8000_0008, 0, EBX, 12, "AMD IBPB"), + (0x8000_0008, 0, EBX, 14, "AMD IBRS"), + (0x8000_0008, 0, EBX, 15, "AMD STIBP"), + (0x8000_0008, 0, EBX, 24, "AMD SSBD"), + (0x8000_0008, 0, EBX, 25, "VIRT_SSBD"), + ]; + for (leaf, subleaf, reg, bit, feature) in hidden { + let value = cpuid(leaf, subleaf)[reg]; + assert_eq!(value & (1 << bit), 0, "{feature} is exposed"); + } + + const XCR0_ALLOWED: u64 = 0b1110_0111; + let xcr0 = cpuid(0xD, 0); + let xcr0 = u64::from(xcr0[EAX]) | u64::from(xcr0[EDX]) << 32; + assert_eq!( + xcr0 & !XCR0_ALLOWED, + 0, + "XCR0 allows state beyond x87, SSE, AVX, and AVX-512" + ); + assert_eq!(cpuid(0xD, 1)[ECX], 0, "IA32_XSS allows supervisor state"); + + // Each kept feature the host supports must reach the guest. Invariant TSC is not + // checked because a nested Hyper-V hides it from child partitions. + let kept = [ + (1, 0, ECX, 0, "SSE3"), + (1, 0, ECX, 1, "PCLMULQDQ"), + (1, 0, ECX, 9, "SSSE3"), + (1, 0, ECX, 12, "FMA"), + (1, 0, ECX, 13, "CMPXCHG16B"), + (1, 0, ECX, 17, "PCID"), + (1, 0, ECX, 19, "SSE4.1"), + (1, 0, ECX, 20, "SSE4.2"), + (1, 0, ECX, 22, "MOVBE"), + (1, 0, ECX, 23, "POPCNT"), + (1, 0, ECX, 25, "AES"), + (1, 0, ECX, 26, "XSAVE"), + (1, 0, ECX, 28, "AVX"), + (1, 0, ECX, 29, "F16C"), + (1, 0, ECX, 30, "RDRAND"), + (7, 0, EBX, 0, "FSGSBASE"), + (7, 0, EBX, 3, "BMI1"), + (7, 0, EBX, 5, "AVX2"), + (7, 0, EBX, 7, "SMEP"), + (7, 0, EBX, 8, "BMI2"), + (7, 0, EBX, 9, "ERMS"), + (7, 0, EBX, 10, "INVPCID"), + (7, 0, EBX, 16, "AVX512F"), + (7, 0, EBX, 17, "AVX512DQ"), + (7, 0, EBX, 18, "RDSEED"), + (7, 0, EBX, 19, "ADX"), + (7, 0, EBX, 20, "SMAP"), + (7, 0, EBX, 21, "AVX512_IFMA"), + (7, 0, EBX, 23, "CLFLUSHOPT"), + (7, 0, EBX, 28, "AVX512CD"), + (7, 0, EBX, 30, "AVX512BW"), + (7, 0, EBX, 31, "AVX512VL"), + (7, 0, ECX, 1, "AVX512_VBMI"), + (7, 0, ECX, 2, "UMIP"), + (7, 0, ECX, 6, "AVX512_VBMI2"), + (7, 0, ECX, 8, "GFNI"), + (7, 0, ECX, 9, "VAES"), + (7, 0, ECX, 10, "VPCLMULQDQ"), + (7, 0, ECX, 11, "AVX512_VNNI"), + (7, 0, ECX, 12, "AVX512_BITALG"), + (7, 0, ECX, 14, "AVX512_VPOPCNTDQ"), + (7, 0, ECX, 22, "RDPID"), + (7, 0, EDX, 4, "FSRM"), + (7, 0, EDX, 8, "AVX512_VP2INTERSECT"), + (7, 0, EDX, 10, "MD_CLEAR"), + (7, 0, EDX, 23, "AVX512_FP16"), + (7, 1, EAX, 4, "AVX_VNNI"), + (7, 1, EAX, 5, "AVX512_BF16"), + (0xD, 1, EAX, 0, "XSAVEOPT"), + (0xD, 1, EAX, 1, "XSAVEC"), + (0x8000_0001, 0, ECX, 0, "LAHF/SAHF"), + (0x8000_0001, 0, ECX, 5, "LZCNT"), + (0x8000_0001, 0, ECX, 6, "SSE4A"), + (0x8000_0001, 0, ECX, 8, "PREFETCHW"), + (0x8000_0001, 0, ECX, 11, "XOP"), + (0x8000_0001, 0, ECX, 16, "FMA4"), + (0x8000_0001, 0, EDX, 26, "1 GB pages"), + (0x8000_0001, 0, EDX, 27, "RDTSCP"), + ]; + for (leaf, subleaf, reg, bit, feature) in kept { + // SAFETY: CPUID has no memory effects. + let host = unsafe { std::arch::x86_64::__cpuid_count(leaf, subleaf) }; + let host = [host.eax, host.ebx, host.ecx, host.edx][reg]; + if host & (1 << bit) != 0 { + let value = cpuid(leaf, subleaf)[reg]; + assert_ne!(value & (1 << bit), 0, "{feature} is hidden"); + } + } + } + #[test] fn snapshot_msr_values_survive_full_in_memory_lifecycle() { let mut source = SandboxBuilder::from_file(simple_guest_as_pathbuf()) diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs b/src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs index a87eb1999..126cb0c17 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs @@ -27,7 +27,7 @@ pub(in crate::sandbox::snapshot) const MT_TRANSPORT_CURRENT: &str = MT_TRANSPORT /// ABI version for the snapshot memory blob. Bumped when the /// host-guest contract for the snapshot bytes changes. See /// docs/snapshot-versioning.md. -pub(in crate::sandbox::snapshot) const SNAPSHOT_ABI_VERSION: u32 = 11; +pub(in crate::sandbox::snapshot) const SNAPSHOT_ABI_VERSION: u32 = 12; /// OCI standard annotation key for a manifest's tag inside an image /// index. Set on the manifest descriptor in `index.json`, not on the diff --git a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs index 970bbf3b5..0dd295409 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs @@ -336,7 +336,7 @@ fn layered_writer_preserves_blobs_and_transport_geometry() { manifest["artifactType"], "application/vnd.hyperlight.snapshot.config.v6+json" ); - assert_eq!(config["abi_version"], 11); + assert_eq!(config["abi_version"], 12); let blobs = snapshot.snapshot_memory().blobs().collect::>(); assert_eq!(layers.len(), blobs.len() + 1); for (index, blob) in blobs.iter().enumerate() { @@ -2520,7 +2520,7 @@ fn legacy_config_versions_rejected() { &path, OciTag::new("latest").unwrap(), )); - assert_err_contains(err, "incompatible with snapshot ABI 11"); + assert_err_contains(err, "incompatible with snapshot ABI 12"); } } diff --git a/src/hyperlight_host/src/sandbox/snapshot/tripwires.rs b/src/hyperlight_host/src/sandbox/snapshot/tripwires.rs index f7a56dc7a..7f0f5bbbc 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/tripwires.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/tripwires.rs @@ -18,7 +18,7 @@ use super::file::{ SNAPSHOT_ABI_VERSION, }; -const EXPECTED_ABI_VERSION: u32 = 11; +const EXPECTED_ABI_VERSION: u32 = 12; const EXPECTED_MT_CONFIG: &str = "application/vnd.hyperlight.snapshot.config.v6+json"; const EXPECTED_MT_SNAPSHOT: &str = "application/vnd.hyperlight.snapshot.memory.v2"; const EXPECTED_MT_TRANSPORT: &str = "application/vnd.hyperlight.snapshot.transport.v1"; diff --git a/src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs b/src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs index c1b3f149b..4fc6ca7f6 100644 --- a/src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs +++ b/src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs @@ -8,7 +8,7 @@ //! publish. See `docs/snapshot-versioning.md`. /// Goldens version, a `vMAJOR.MINOR` string. -pub(crate) const GOLDENS_VERSION: &str = "v11.0"; +pub(crate) const GOLDENS_VERSION: &str = "v12.0"; /// Old majors kept loadable through a compatibility path, verified /// alongside `GOLDENS_VERSION`. A backwards-compatible break (Option 2) diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index 99da90ed0..1d48d6d1e 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -1312,6 +1312,19 @@ fn read_tsc() -> u64 { unsafe { core::arch::x86_64::_rdtsc() } } +/// Returns EAX, EBX, ECX and EDX as little-endian bytes. +#[guest_function("Cpuid")] +#[cfg(target_arch = "x86_64")] +#[allow(unused_unsafe)] +fn cpuid(leaf: u32, subleaf: u32) -> Vec { + // SAFETY: CPUID has no memory effects. Unsupported leaves return defined values. + let r = unsafe { core::arch::x86_64::__cpuid_count(leaf, subleaf) }; + [r.eax, r.ebx, r.ecx, r.edx] + .iter() + .flat_map(|reg| reg.to_le_bytes()) + .collect() +} + #[guest_function("ReadMSR")] #[cfg(target_arch = "x86_64")] fn read_msr(msr: u32) -> u64 {