From 85fcac0976a129cfe0ea8ab624c9ecd05ffe95eb Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Sun, 1 Nov 2026 22:12:20 -0800 Subject: [PATCH 1/4] cpu_profile: compute the effective CPUID of nested virtualization Part of microsoft/nvx#256 and microsoft/nvx#1178. A microVM's effective CPUID is a pure function of its CPU profile, its topology, and the time ABI's identity leaves, and every CPU profile pins the virtualization extensions clear, as the derivation policy requires. A microVM that gives its guest nested virtualization must expose Intel's VMX, CPUID.1:ECX[5], so the effective CPUID needs a fourth input: whether the VM has nested virtualization. Add cpu_profile::nested_virtualization_cpuid(vendor), the CPUID result that exposes hardware virtualization to a guest of a profile of the vendor's CPUs: VMX, set under its mask, on Intel. AMD's guests would need SVM, whose feature leaf 0x8000000A a guest hypervisor reads, and no profile describes that leaf, so an AMD profile refuses nested virtualization with E_PROFILE_UNSUPPORTED rather than exposing SVM. EffectiveCpuid::with_nested_virtualization(vendor) sets the bit over the profile's pinned zero, and nothing else: the profile documents and their digests are unchanged, and so are the identity leaves, so leaf 0x4000000A still reads zero. The VMX capability MSRs, which describe the controls that a guest hypervisor may use, stay the backend's. virt::time_abi::cpuid::effective_cpuid() takes nested_virt, and the worker passes the VM's HypervisorConfig::nested_virt. The microVM machine still refuses nested virtualization, so every effective CPUID is the one of before; the tests and the MSHV backend, whose time ABI partitions never have it, pass false. A snapshot records its VM's effective CPUID, and a restore recomputes it and requires it to match. A VM with nested virtualization is never captured, which a later commit enforces, so no record sets VMX, and a record that did would not match the one that a restore recomputes without it. Tests: - cpu_profile: nested_virtualization_sets_only_vmx_on_intel_profiles checks that the result stays canonical and differs from the profile's in that bit alone; nested_virtualization_changes_the_effective_cpuid_record that a VM with it and one without it never match; and nested_virtualization_refuses_amd_profiles the AMD refusal. - virt: nested_virtualization_adds_vmx_to_intel_profiles_only computes the effective CPUID of every pinned profile with and without it, which still satisfies the time ABI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openvmm/openvmm_core/src/worker/dispatch.rs | 4 + .../src/worker/dispatch/time_abi.rs | 11 +- vmm_core/cpu_profile/src/effective.rs | 146 ++++++++++++++++++ vmm_core/cpu_profile/src/lib.rs | 6 +- vmm_core/virt/src/time_abi/cpuid.rs | 60 ++++++- vmm_core/virt_mshv/src/x86_64/time_abi.rs | 3 +- vmm_core/virt_whp/src/time_abi.rs | 7 +- 7 files changed, 223 insertions(+), 14 deletions(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index e97ba414860..c737eb8a488 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -1331,9 +1331,13 @@ impl InitializedVm { cfg.microvm.restored_host_profile.as_ref(), ¶meters.hooks, )?; + // Nested virtualization sets VMX over the profile's surface; a + // restore never has it, so a snapshot's effective CPUID never + // does either. let effective_cpuid = Arc::new(virt::time_abi::cpuid::effective_cpuid( &profile, &processor_topology, + cfg.hypervisor.nested_virt, )?); let (msrs, config) = time_abi::partition_config(&profile, &effective_cpuid); ( diff --git a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs index 019e4a3b850..e7f3f2c59d0 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs @@ -1457,7 +1457,7 @@ mod tests { fn test_config(vp_count: u32) -> (Arc, TimeAbiConfig, EffectiveCpuid) { let profile = test_profile(); let effective = - effective_cpuid(profile, &topology(vp_count, X2ApicState::Supported)).unwrap(); + effective_cpuid(profile, &topology(vp_count, X2ApicState::Supported), false).unwrap(); let (msrs, config) = partition_config(&partition_profile(profile), &effective); (msrs, config, effective) } @@ -1594,7 +1594,8 @@ mod tests { for profile in cpu_profile::pinned_profiles() { for vp_count in [1, 2, 8] { for x2apic in [X2ApicState::Unsupported, X2ApicState::Supported] { - let effective = effective_cpuid(profile, &topology(vp_count, x2apic)).unwrap(); + let effective = + effective_cpuid(profile, &topology(vp_count, x2apic), false).unwrap(); let (_, config) = partition_config(&partition_profile(profile), &effective); assert_eq!(config.cpu_profile.id(), profile.id()); @@ -1630,7 +1631,8 @@ mod tests { for profile in cpu_profile::pinned_profiles() { for vp_count in [1, 2, 8, 64] { for x2apic in [X2ApicState::Unsupported, X2ApicState::Supported] { - let effective = effective_cpuid(profile, &topology(vp_count, x2apic)).unwrap(); + let effective = + effective_cpuid(profile, &topology(vp_count, x2apic), false).unwrap(); let (_, config) = partition_config(&partition_profile(profile), &effective); check_presented(profile, &effective, config.cpuid.leaves().to_vec()) .unwrap_or_else(|err| { @@ -1691,7 +1693,8 @@ mod tests { assert!(profile.lookup(0, 0)[0] < 0x1f, "{}", profile.id()); for vp_count in [1, 2, 8] { let effective = - effective_cpuid(profile, &topology(vp_count, X2ApicState::Supported)).unwrap(); + effective_cpuid(profile, &topology(vp_count, X2ApicState::Supported), false) + .unwrap(); let subleaves = |function| { effective .results() diff --git a/vmm_core/cpu_profile/src/effective.rs b/vmm_core/cpu_profile/src/effective.rs index 4bb752e5313..fd27c2dc525 100644 --- a/vmm_core/cpu_profile/src/effective.rs +++ b/vmm_core/cpu_profile/src/effective.rs @@ -14,6 +14,7 @@ use crate::profile::VM_OWNED_LEAVES; use crate::profile::describe_leaf; use crate::profile::find; use crate::profile::vm_owned_bits; +use crate::vendor::CpuVendor; use serde::Deserialize; use std::cmp::Ordering; @@ -76,6 +77,34 @@ pub const fn x2apic_cpuid(x2apic: bool) -> CpuidResult { CpuidResult::new(0x1, [0, 0, if x2apic { X2APIC } else { 0 }, 0]).masked([0, 0, X2APIC, 0]) } +/// `CPUID.1:ECX[5]`: VMX, Intel's hardware virtualization. +const VMX: u32 = 1 << 5; + +/// Returns the CPUID result through which a VM with nested virtualization +/// exposes hardware virtualization to a guest of a profile of `vendor`'s +/// CPUs: VMX, `CPUID.1:ECX[5]`, set, on Intel. +/// +/// Every profile pins the virtualization extensions clear, as the derivation +/// policy requires, so a profile alone never exposes them, and only +/// [`EffectiveCpuid::with_nested_virtualization`] sets this bit over its +/// pinned value. VMX's capability MSRs, which describe the VMX controls that +/// a guest hypervisor may use, are the backend's. +/// +/// Fails with `E_PROFILE_UNSUPPORTED` for a profile of AMD's CPUs, whose +/// guests would need SVM: no profile describes SVM's feature leaf, +/// `0x8000000A`, which a guest hypervisor reads, so nested virtualization +/// does not expose SVM. +pub fn nested_virtualization_cpuid(vendor: CpuVendor) -> Result { + match vendor { + CpuVendor::Intel => Ok(CpuidResult::new(0x1, [0, 0, VMX, 0]).masked([0, 0, VMX, 0])), + CpuVendor::Amd => Err(ProfileError::new( + ProfileErrorCode::ProfileUnsupported, + "nested virtualization requires an Intel CPU profile: it exposes VMX, but no \ + profile of AMD's CPUs describes SVM's feature leaf 0x8000000A", + )), + } +} + /// The effective guest CPUID of a VM. /// /// It defines every bit of every leaf and subleaf the guest can observe, @@ -241,6 +270,40 @@ impl EffectiveCpuid { Self::decode(bytes) } + /// Returns this effective CPUID, of a profile of `vendor`'s CPUs, with + /// the hardware virtualization of nested virtualization + /// ([`nested_virtualization_cpuid`]) set over the profile's pinned + /// value: VMX, on Intel. + /// + /// A snapshot's effective CPUID never has it, since no VM with nested + /// virtualization is captured, so a destination that recomputes the + /// effective CPUID of a snapshot's VM without it would also reject one + /// that did ([`Self::check_matches`]). Fails with + /// `E_PROFILE_UNSUPPORTED` on AMD, and with `E_CPU_SURFACE` if leaf 1 is + /// missing. + pub fn with_nested_virtualization(mut self, vendor: CpuVendor) -> Result { + let overlay = nested_virtualization_cpuid(vendor)?; + let Some(entry) = self + .entries + .iter_mut() + .find(|entry| entry.key() == (overlay.function, overlay.index)) + else { + return Err(ProfileError::new( + ProfileErrorCode::CpuSurface, + format!( + "the effective CPUID has no {} for nested virtualization", + describe_leaf(overlay.function, overlay.index) + ), + )); + }; + for register in 0..4 { + entry.value[register].0 = (entry.value[register].0 & !overlay.mask[register]) + | (overlay.result[register] & overlay.mask[register]); + entry.mask[register].0 |= overlay.mask[register]; + } + Ok(self) + } + /// Checks that this effective CPUID, recomputed on the destination, /// equals the `recorded` one (`E_CPU_SURFACE`, naming every difference). pub fn check_matches(&self, recorded: &Self) -> Result<(), ProfileError> { @@ -747,6 +810,89 @@ mod tests { ); } + /// Nested virtualization sets VMX, and nothing else, over every Intel + /// profile's pinned zero, and the result stays a canonical record. + #[test] + fn nested_virtualization_sets_only_vmx_on_intel_profiles() { + for profile in crate::pinned_profiles() + .iter() + .filter(|profile| profile.cpu_vendor() == CpuVendor::Intel) + { + let effective = profile + .effective_cpuid(&vm_leaves(profile, 2), &identity()) + .unwrap(); + assert_eq!(effective.lookup(1, 0)[2] & VMX, 0, "{}", profile.id()); + let nested = effective + .clone() + .with_nested_virtualization(CpuVendor::Intel) + .unwrap(); + assert_eq!(nested.lookup(1, 0)[2] & VMX, VMX, "{}", profile.id()); + let differences = nested + .results() + .zip(effective.results()) + .filter(|(nested, effective)| nested != effective) + .map(|(nested, effective)| { + ( + nested.function, + nested.index, + [0, 1, 2, 3] + .map(|register| nested.result[register] ^ effective.result[register]), + nested.mask == effective.mask, + ) + }) + .collect::>(); + assert_eq!( + differences, + [(1, None, [0, 0, VMX, 0], true)], + "{}", + profile.id() + ); + assert_eq!(EffectiveCpuid::decode(&nested.encode()).unwrap(), nested); + } + } + + /// A snapshot's effective CPUID never has VMX, so a VM with nested + /// virtualization and a VM without it never match. + #[test] + fn nested_virtualization_changes_the_effective_cpuid_record() { + let profile = profile(ICELAKE); + let effective = profile + .effective_cpuid(&vm_leaves(profile, 2), &identity()) + .unwrap(); + let nested = effective + .clone() + .with_nested_virtualization(CpuVendor::Intel) + .unwrap(); + let error = nested.check_matches(&effective).unwrap_err(); + assert_eq!(error.code, ProfileErrorCode::CpuSurface); + assert!(error.message.contains("CPUID 0x1 is"), "{error}"); + effective.check_matches(&nested).unwrap_err(); + } + + /// No profile of AMD's CPUs describes SVM's feature leaf, so nested + /// virtualization refuses them. + #[test] + fn nested_virtualization_refuses_amd_profiles() { + let error = nested_virtualization_cpuid(CpuVendor::Amd).unwrap_err(); + assert_eq!(error.code, ProfileErrorCode::ProfileUnsupported); + assert!(error.message.contains("0x8000000A"), "{error}"); + let profile = amd_host_profile(); + let effective = profile + .effective_cpuid(&vm_leaves(&profile, 2), &identity()) + .unwrap(); + assert_eq!( + effective + .with_nested_virtualization(CpuVendor::Amd) + .unwrap_err() + .code, + ProfileErrorCode::ProfileUnsupported + ); + assert_eq!( + nested_virtualization_cpuid(CpuVendor::Intel).unwrap(), + CpuidResult::new(1, [0, 0, VMX, 0]).masked([0, 0, VMX, 0]) + ); + } + /// `encode` serializes directly, which is canonical only while the /// document and its entries declare their fields in key order. #[test] diff --git a/vmm_core/cpu_profile/src/lib.rs b/vmm_core/cpu_profile/src/lib.rs index bf4f02d9158..952046ecd97 100644 --- a/vmm_core/cpu_profile/src/lib.rs +++ b/vmm_core/cpu_profile/src/lib.rs @@ -11,7 +11,10 @@ //! ([`pinned`]), immutable once released, and identified by ID and SHA-256 //! digest. A VM's effective CPUID ([`CpuProfile::effective_cpuid`]) is a pure //! function of its profile, its topology, and the time ABI's identity -//! leaves; OpenVMM builds it in `virt::time_abi::cpuid::effective_cpuid`. +//! leaves, and of whether the VM has nested virtualization, which sets VMX +//! over the profile's pinned value +//! ([`EffectiveCpuid::with_nested_virtualization`]); OpenVMM builds it in +//! `virt::time_abi::cpuid::effective_cpuid`. //! //! - **Selection:** [`select`] maps `--cpu-profile ` and the host //! CPU ([`HostCpuSignature`]) to a pinned profile. A host profile serves a @@ -115,6 +118,7 @@ pub use derive::supports_host_profiles; pub use effective::CpuidResult; pub use effective::EFFECTIVE_CPUID_SCHEMA; pub use effective::EffectiveCpuid; +pub use effective::nested_virtualization_cpuid; pub use effective::x2apic_cpuid; pub use error::ProfileError; pub use error::ProfileErrorCode; diff --git a/vmm_core/virt/src/time_abi/cpuid.rs b/vmm_core/virt/src/time_abi/cpuid.rs index b8950e678f7..abe4cde18fd 100644 --- a/vmm_core/virt/src/time_abi/cpuid.rs +++ b/vmm_core/virt/src/time_abi/cpuid.rs @@ -30,11 +30,16 @@ impl From<&CpuidLeaf> for CpuidResult { /// Builds the effective CPUID of a partition with `profile` and `topology`: /// the profile completed with OpenVMM's topology leaves, each extended /// topology leaf's terminating subleaf, the APIC mode, and the time ABI's -/// identity and explicit zero leaves. Fails with `E_CPU_SURFACE` if they do -/// not complete the profile exactly. +/// identity and explicit zero leaves, and, when `nested_virt` is set, the +/// hardware virtualization that nested virtualization exposes +/// ([`EffectiveCpuid::with_nested_virtualization`]). Fails with +/// `E_CPU_SURFACE` if they do not complete the profile exactly, and with +/// `E_PROFILE_UNSUPPORTED` for nested virtualization under a profile of AMD's +/// CPUs. pub fn effective_cpuid( profile: &CpuProfile, topology: &ProcessorTopology, + nested_virt: bool, ) -> Result { let mut topology_leaves = Vec::new(); crate::x86::topology::topology_cpuid( @@ -62,7 +67,12 @@ pub fn effective_cpuid( .map(CpuidResult::from) .chain(super::identity::identity_zero_cpuid_leaves().map(|leaf| CpuidResult::from(&leaf))) .collect(); - Ok(profile.effective_cpuid(&vm, &identity)?) + let effective = profile.effective_cpuid(&vm, &identity)?; + Ok(if nested_virt { + effective.with_nested_virtualization(profile.cpu_vendor())? + } else { + effective + }) } /// Returns the CPUID results a backend programs for `effective` @@ -113,7 +123,7 @@ mod tests { for profile in cpu_profile::pinned_profiles() { for vp_count in [1, 2, 8] { for x2apic in [X2ApicState::Unsupported, X2ApicState::Supported] { - let effective = effective_cpuid(profile, &topology(vp_count, x2apic)) + let effective = effective_cpuid(profile, &topology(vp_count, x2apic), false) .unwrap_or_else(|error| panic!("{}: {error}", profile.id())); identity::check_time_bits( &mut |leaf, subleaf| effective.lookup(leaf, subleaf), @@ -148,7 +158,7 @@ mod tests { for vp_count in [1, 2, 8] { let topology = topology(vp_count, X2ApicState::Supported); let socket = topology.reserved_vps_per_socket(); - let effective = effective_cpuid(profile, &topology).unwrap(); + let effective = effective_cpuid(profile, &topology, false).unwrap(); let mut levels = Vec::new(); for subleaf in 0..16 { let eax = CacheParametersEax::from(effective.lookup(leaf, subleaf)[0]); @@ -179,10 +189,48 @@ mod tests { } } + /// Nested virtualization adds VMX, `CPUID.1:ECX[5]`, to every Intel + /// profile's effective CPUID, which still satisfies the time ABI, and a + /// backend programs it under its mask; AMD profiles refuse it. + #[test] + fn nested_virtualization_adds_vmx_to_intel_profiles_only() { + const VMX: u32 = 1 << 5; + for profile in cpu_profile::pinned_profiles() { + let topology = topology(2, X2ApicState::Unsupported); + let result = effective_cpuid(profile, &topology, true); + if profile.cpu_vendor() == CpuVendor::Amd { + assert_eq!( + result.unwrap_err().code, + TimeAbiCode::ProfileUnsupported, + "{}", + profile.id() + ); + continue; + } + let nested = result.unwrap_or_else(|error| panic!("{}: {error}", profile.id())); + let plain = effective_cpuid(profile, &topology, false).unwrap(); + assert_eq!(plain.lookup(1, 0)[2] & VMX, 0, "{}", profile.id()); + assert_eq!(nested.lookup(1, 0)[2] & VMX, VMX, "{}", profile.id()); + identity::check_time_bits(&mut |leaf, subleaf| nested.lookup(leaf, subleaf), true) + .unwrap_or_else(|error| panic!("{}: {error}", profile.id())); + identity::check_identity(&mut |leaf, subleaf| nested.lookup(leaf, subleaf), 2) + .unwrap_or_else(|error| panic!("{}: {error}", profile.id())); + let leaf = backend_cpuid(&nested) + .leaves() + .iter() + .find(|leaf| leaf.function == 1) + .copied() + .unwrap(); + assert_eq!(leaf.result[2] & VMX, VMX, "{}", profile.id()); + assert_eq!(leaf.mask[2] & VMX, VMX, "{}", profile.id()); + } + } + #[test] fn backend_cpuid_unmasks_only_the_per_vp_apic_bits() { for profile in cpu_profile::pinned_profiles() { - let effective = effective_cpuid(profile, &topology(2, X2ApicState::Supported)).unwrap(); + let effective = + effective_cpuid(profile, &topology(2, X2ApicState::Supported), false).unwrap(); let table = backend_cpuid(&effective); assert_eq!( table.leaves().len(), diff --git a/vmm_core/virt_mshv/src/x86_64/time_abi.rs b/vmm_core/virt_mshv/src/x86_64/time_abi.rs index 90cd8c67847..77de8a861de 100644 --- a/vmm_core/virt_mshv/src/x86_64/time_abi.rs +++ b/vmm_core/virt_mshv/src/x86_64/time_abi.rs @@ -1909,7 +1909,8 @@ mod hw { fn host_profile_cpuid( topology: &vm_topology::processor::ProcessorTopology, ) -> (cpu_profile::EffectiveCpuid, CpuidLeafSet) { - let effective = virt::time_abi::cpuid::effective_cpuid(&host_profile(), topology).unwrap(); + let effective = + virt::time_abi::cpuid::effective_cpuid(&host_profile(), topology, false).unwrap(); let table = virt::time_abi::cpuid::backend_cpuid(&effective); (effective, table) } diff --git a/vmm_core/virt_whp/src/time_abi.rs b/vmm_core/virt_whp/src/time_abi.rs index 43238a6f0ec..e618e570228 100644 --- a/vmm_core/virt_whp/src/time_abi.rs +++ b/vmm_core/virt_whp/src/time_abi.rs @@ -1608,6 +1608,7 @@ mod tests { let effective = virt::time_abi::cpuid::effective_cpuid( cpu_profile::pinned(profile).unwrap(), &topology, + false, ) .unwrap(); let config = virt::time_abi::cpuid::backend_cpuid(&effective); @@ -1636,7 +1637,8 @@ mod tests { fn profile_programmed(profile: &str, vp_count: u32, noise: bool) -> Self { let topology = test_cpuid::topology(vp_count, X2ApicState::Supported); let pinned = cpu_profile::pinned(profile).unwrap(); - let effective = virt::time_abi::cpuid::effective_cpuid(pinned, &topology).unwrap(); + let effective = + virt::time_abi::cpuid::effective_cpuid(pinned, &topology, false).unwrap(); let programming = test_cpuid::profile_programming(pinned, vp_count); let whp = FakeWhp { results: programming.results, @@ -2472,7 +2474,8 @@ mod whp_tests { } }; let topology = test_cpuid::topology(vp_count, X2ApicState::Supported); - let effective = virt::time_abi::cpuid::effective_cpuid(&profile, &topology).unwrap(); + let effective = + virt::time_abi::cpuid::effective_cpuid(&profile, &topology, false).unwrap(); let config = TimeAbiConfig { cpuid: Arc::new(virt::time_abi::cpuid::backend_cpuid(&effective)), msrs: Arc::new(TimeAbiMsrs::new()), From 29e7cac0ebe695fc3f756277cd08e526127430fc Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Sun, 1 Nov 2026 22:12:21 -0800 Subject: [PATCH 2/4] virt_whp: serve IA32_XSS reads on time ABI partitions Part of microsoft/nvx#256 and microsoft/nvx#1178. WHP keeps IA32_XSS (0xda0) in the virtual processor, and takes the guest's writes of it, but on a time ABI partition, whose XSAVE features enable no supervisor state, it sends a read of it to OpenVMM's MSR exit. The time ABI serves only its identity MSRs there, so the read fell through to the handler of unknown MSRs, which faults it. Every pinned CPU profile enumerates XSAVES (CPUID.(EAX=0DH,ECX=1):EAX[3]), so the guest may read the MSR, and on KVM it does: Linux writes XSS at boot, and once the guest kernel builds KVM, kvm_x86_vendor_init() reads it back with an unchecked RDMSR when KVM initializes. On WHP, the guest of a microVM with nested virtualization, the first whose KVM initializes, therefore logged an unchecked MSR access, which its time ABI watcher reports as G_UNCHECKED_MSR before it powers the guest off, and a read through /dev/cpu/N/msr in a default microVM failed with EIO. handle_time_abi_msr() now completes a read of IA32_XSS with the VP's Xss register, the last value that the guest wrote, and advances RIP past RDMSR, as it does for the identity MSRs. A write still goes to WHP, which takes it. The whp crate names the register as Register64::Xss (WHvX64RegisterXss). WHP's time ABI partitions now serve IA32_XSS as KVM's do. MSHV's still fault the read, which no guest notices: MSHV never gives a microVM nested virtualization, so the guest's KVM never starts there. NVX's nested-virtualization scenario reads the MSR in every microVM on KVM and WHP, and the nested one starts the guest's KVM, which reads it too. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vm/whp/src/x64.rs | 1 + vmm_core/virt_whp/src/time_abi.rs | 28 +++++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/vm/whp/src/x64.rs b/vm/whp/src/x64.rs index 10e328d1434..f342dd38bcd 100644 --- a/vm/whp/src/x64.rs +++ b/vm/whp/src/x64.rs @@ -41,6 +41,7 @@ pub enum Register64 { Dr6 = abi::WHvX64RegisterDr6.0, Dr7 = abi::WHvX64RegisterDr7.0, XCr0 = abi::WHvX64RegisterXCr0.0, + Xss = abi::WHvX64RegisterXss.0, Tsc = abi::WHvX64RegisterTsc.0, TscAdjust = abi::WHvX64RegisterTscAdjust.0, diff --git a/vmm_core/virt_whp/src/time_abi.rs b/vmm_core/virt_whp/src/time_abi.rs index e618e570228..0910a04470d 100644 --- a/vmm_core/virt_whp/src/time_abi.rs +++ b/vmm_core/virt_whp/src/time_abi.rs @@ -1257,6 +1257,31 @@ impl WhpProcessor<'_> { } else { MsrAccess::Read }; + // RDMSR and WRMSR are two bytes long. A faulting access does not + // retire, so RIP stays. + let rip = exit.vp_context.Rip.wrapping_add(2); + // WHP keeps `IA32_XSS` in the VP, and takes the guest's writes of it, + // but sends a read of it here on a time ABI partition, whose XSAVE + // features enable no supervisor state. Every profile enumerates + // XSAVES, so a read must return what the guest wrote, as it does on + // KVM: Linux writes the MSR at boot, and its KVM reads it back when it + // initializes, under nested virtualization. + if info.MsrNumber == x86defs::X86X_MSR_XSS && access == MsrAccess::Read { + let value = self + .current_whp() + .get_register(whp::Register64::Xss) + .for_op("read IA32_XSS for a time ABI MSR read")?; + whp::set_registers!( + self.current_whp(), + [ + (whp::Register64::Rax, value & 0xffff_ffff), + (whp::Register64::Rdx, value >> 32), + (whp::Register64::Rip, rip), + ] + ) + .for_op("complete a time ABI IA32_XSS read")?; + return Ok(true); + } let Some(outcome) = serve_msr( &time_abi.msrs, time_abi.cpu_profile.cpu_vendor(), @@ -1273,9 +1298,6 @@ impl WhpProcessor<'_> { ?outcome, "time ABI MSR access" ); - // RDMSR and WRMSR are two bytes long. A faulting access does not - // retire, so RIP stays. - let rip = exit.vp_context.Rip.wrapping_add(2); match outcome { MsrOutcome::Read(value) => whp::set_registers!( self.current_whp(), From 16e3207e5db4a58fe6ac40ffc43cc45414ba3654 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 2 Nov 2026 00:15:55 -0800 Subject: [PATCH 3/4] virt_whp: let time ABI partitions have nested virtualization Part of microsoft/nvx#256 and microsoft/nvx#1178. The WHP time ABI refused nested virtualization in validate_partition(), beside isolation, the Hyper-V guest interface, and the user-mode APIC. Those three conflict with the time ABI's identity: they enable WHP's synthetic features, which serve the identity leaves and MSRs natively, or VTL2. Nested virtualization needs none of them: WHP offers it as WHvPartitionPropertyCodeNestedVirtualization and the NestedVirtSupport bit of the second processor-feature bank, and a partition with it but no synthetic features still reports no hypervisor leaf of its own. - validate_partition() no longer takes nested_virt, and refuses only isolation, the Hyper-V guest interface, and the user-mode APIC. - WhpPartition::new() sets the NestedVirtualization property before WhpTimeAbi::configure() writes the feature banks, so that the banks apply to a nested partition, and, for a time ABI partition, requires WHP's own NestedVirtSupport, failing with NestedVirtUnsupported where WHP does not offer it, as it already did where WHP does not know the property. - WhpTimeAbi::configure() takes nested_virt, and offers NestedVirtSupport in the partition's bank 1 beside the profile's features. The effective CPUID sets VMX for such a partition, and the CPUID exits keep serving it, so VP 0 sees VMX; WHP still answers the identity range past the exits, nested virtualization's leaves included, with zeros. The microVM machine still refuses nested virtualization; the next commit allows it. Tests: - partition_validation_rejects_unsupported_configurations no longer passes nested_virt. - The hardware tests share partition_with(), check_static_obligations(), and check_host_profile_configures_the_partition() between ordinary and nested partitions. The new ignored tests nested_partition_meets_the_static_obligations and host_profile_configures_a_nested_partition require that a nested partition meets the time ABI's static obligations, sees VMX on VP 0, and offers NestedVirtSupport, and skip where the host's CPU profile is not Intel's, whose effective CPUID refuses nested virtualization, or where WHP offers no nested virtualization. Both pass on an Intel Alder Lake host with Windows 11 and nested virtualization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/virt_whp/src/lib.rs | 42 ++++++---- vmm_core/virt_whp/src/time_abi.rs | 122 ++++++++++++++++++++++-------- 2 files changed, 119 insertions(+), 45 deletions(-) diff --git a/vmm_core/virt_whp/src/lib.rs b/vmm_core/virt_whp/src/lib.rs index 05f6d6e11de..3fb0c6a69e0 100644 --- a/vmm_core/virt_whp/src/lib.rs +++ b/vmm_core/virt_whp/src/lib.rs @@ -1018,12 +1018,7 @@ impl virt::Hypervisor for Whp { #[cfg(guest_arch = "x86_64")] if config.time_abi.is_some() { - time_abi::validate_partition( - isolation, - config.hv_config.is_some(), - nested_virt, - user_mode_apic, - )?; + time_abi::validate_partition(isolation, config.hv_config.is_some(), user_mode_apic)?; } let vtl0 = VtlPartition::new( @@ -1639,15 +1634,8 @@ impl VtlPartition { )) .for_op("set processor count")?; - #[cfg(guest_arch = "x86_64")] - let time_abi = config - .time_abi - .as_ref() - .map(|time_abi| { - time_abi::WhpTimeAbi::configure(time_abi, &mut whp_config, &mut extended_exits) - }) - .transpose()?; - + // Nested virtualization goes first, so that a time ABI partition's + // feature banks, which then offer it, apply to a nested partition. #[cfg(guest_arch = "x86_64")] if nested_virt { match whp_config.set_property(whp::PartitionProperty::NestedVirtualization(true)) { @@ -1659,8 +1647,32 @@ impl VtlPartition { return Err(err).for_op("enable nested virtualization"); } } + // A time ABI partition offers the feature in its banks, which + // WHP offers only on a host that can nest. + if config.time_abi.is_some() + && !whp::capabilities::processor_features() + .for_op("get processor features")? + .bank1 + .is_set(whp::abi::WHV_PROCESSOR_FEATURES1::NestedVirtSupport) + { + return Err(Error::NestedVirtUnsupported); + } } + #[cfg(guest_arch = "x86_64")] + let time_abi = config + .time_abi + .as_ref() + .map(|time_abi| { + time_abi::WhpTimeAbi::configure( + time_abi, + &mut whp_config, + &mut extended_exits, + nested_virt, + ) + }) + .transpose()?; + #[cfg(guest_arch = "x86_64")] { use vm_topology::processor::x86::ApicMode; diff --git a/vmm_core/virt_whp/src/time_abi.rs b/vmm_core/virt_whp/src/time_abi.rs index 0910a04470d..1ecef367343 100644 --- a/vmm_core/virt_whp/src/time_abi.rs +++ b/vmm_core/virt_whp/src/time_abi.rs @@ -8,9 +8,11 @@ //! It meets the time ABI's backend obligations with these WHP mechanisms: //! //! - **Partition.** No Hyper-V guest interface (so no synthetic processor -//! features), the in-hypervisor (offloaded) APIC, no isolation, no VTL2, -//! and no nested virtualization. Without synthetic features WHP reports no -//! hypervisor CPUID leaf and serves no synthetic MSR of its own. +//! features), the in-hypervisor (offloaded) APIC, no isolation, and no +//! VTL2. Without synthetic features WHP reports no hypervisor CPUID leaf +//! and serves no synthetic MSR of its own. Nested virtualization needs +//! none of them, so a time ABI partition may have it: the effective CPUID +//! then sets VMX, and the feature banks offer WHP's nested virtualization. //! - **Processor features.** Both feature banks and the XSAVE features derive //! from the partition's CPU profile ([`profile_features`]), without the //! TSC-deadline timer, `IA32_TSC_ADJUST`, or APERF/MPERF. WHP's default @@ -211,11 +213,15 @@ impl WhpTimeAbi { /// The processor feature banks and the XSAVE features derive from the /// CPU profile `config.cpu_profile` ([`profile_features`]), and /// `config.cpuid`, the complete effective CPUID, becomes the CPUID results - /// ([`cpuid_results`]) and the exits ([`cpuid_exits`]). + /// ([`cpuid_results`]) and the exits ([`cpuid_exits`]). A partition with + /// `nested_virt` also offers WHP's nested virtualization feature, which + /// presents VMX where the effective CPUID sets it; the profile's policy + /// clears VMX, so the profile alone never offers it. pub(crate) fn configure( config: &TimeAbiConfig, whp_config: &mut whp::PartitionConfig, extended_exits: &mut WHV_EXTENDED_VM_EXITS, + nested_virt: bool, ) -> Result { let profile = &config.cpu_profile; @@ -257,6 +263,10 @@ impl WhpTimeAbi { let mut features = available; features.bank0 = WHV_PROCESSOR_FEATURES(derived.banks[0]); features.bank1 = WHV_PROCESSOR_FEATURES1(derived.banks[1]); + if nested_virt { + // The partition checked that WHP offers the feature. + features.bank1 |= WHV_PROCESSOR_FEATURES1::NestedVirtSupport; + } let features = processor_features(features); let xsave = derived.xsave; whp_config @@ -418,12 +428,12 @@ pub(crate) fn unlisted_cpuid( /// Fails unless the partition can carry the time ABI: no isolation, no /// Hyper-V guest interface (whose synthetic processor features would serve -/// the identity CPUID and MSRs natively, and which VTL2 requires), no nested -/// virtualization, and the offloaded APIC. +/// the identity CPUID and MSRs natively, and which VTL2 requires), and the +/// offloaded APIC. Nested virtualization needs none of these, so a time ABI +/// partition may have it. pub(crate) fn validate_partition( isolation: IsolationType, hv_configured: bool, - nested_virt: bool, user_mode_apic: bool, ) -> Result<(), TimeAbiError> { if isolation != IsolationType::None { @@ -436,11 +446,6 @@ pub(crate) fn validate_partition( "the WHP time ABI cannot be combined with the Hyper-V guest interface", )); } - if nested_virt { - return Err(routing_error( - "the WHP time ABI cannot be combined with nested virtualization", - )); - } if user_mode_apic { return Err(TimeAbiError::new( TimeAbiCode::LapicRateUnavailable, @@ -1719,41 +1724,34 @@ mod tests { msrs } + /// The time ABI refuses isolation, the Hyper-V guest interface, and the + /// user-mode APIC. It needs none of them for nested virtualization, which + /// `validate_partition` therefore no longer takes. #[test] fn partition_validation_rejects_unsupported_configurations() { - validate_partition(IsolationType::None, false, false, false).unwrap(); - for (isolation, hv, nested, user_mode_apic, code) in [ + validate_partition(IsolationType::None, false, false).unwrap(); + for (isolation, hv, user_mode_apic, code) in [ ( IsolationType::Vbs, false, false, - false, TimeAbiCode::IdentityRouting, ), ( IsolationType::None, true, false, - false, TimeAbiCode::IdentityRouting, ), ( IsolationType::None, false, true, - false, - TimeAbiCode::IdentityRouting, - ), - ( - IsolationType::None, - false, - false, - true, TimeAbiCode::LapicRateUnavailable, ), ] { assert_eq!( - validate_partition(isolation, hv, nested, user_mode_apic) + validate_partition(isolation, hv, user_mode_apic) .unwrap_err() .code, code @@ -2488,6 +2486,14 @@ mod whp_tests { /// profile, configured as `VtlPartition::new` configures one, or `None` /// on a host outside the pinned generations, where the tests skip. fn partition(vp_count: u32) -> Option { + partition_with(vp_count, false) + } + + /// Returns [`partition`]'s partition, with nested virtualization if + /// `nested_virt` is set, or `None` where the tests skip: on a host + /// outside the pinned generations, and for nested virtualization, on a + /// host whose CPU profile is not Intel's or whose WHP does not offer it. + fn partition_with(vp_count: u32, nested_virt: bool) -> Option { let profile = match cpu_profile::select_auto(&cpu_profile::HostCpuSignature::current()) { Ok(profile) => PartitionProfile::pinned(profile.id()).unwrap(), Err(err) => { @@ -2495,9 +2501,18 @@ mod whp_tests { return None; } }; + if nested_virt && profile.cpu_vendor() != CpuVendor::Intel { + // No profile of AMD's CPUs describes SVM, so they refuse nested + // virtualization with `E_PROFILE_UNSUPPORTED`. + println!( + "skipped: nested virtualization needs an Intel CPU profile, not {}", + profile.id() + ); + return None; + } let topology = test_cpuid::topology(vp_count, X2ApicState::Supported); let effective = - virt::time_abi::cpuid::effective_cpuid(&profile, &topology, false).unwrap(); + virt::time_abi::cpuid::effective_cpuid(&profile, &topology, nested_virt).unwrap(); let config = TimeAbiConfig { cpuid: Arc::new(virt::time_abi::cpuid::backend_cpuid(&effective)), msrs: Arc::new(TimeAbiMsrs::new()), @@ -2507,8 +2522,22 @@ mod whp_tests { whp_config .set_property(whp::PartitionProperty::ProcessorCount(vp_count)) .unwrap(); + if nested_virt { + let offered = whp::capabilities::processor_features() + .unwrap() + .bank1 + .is_set(WHV_PROCESSOR_FEATURES1::NestedVirtSupport); + if !offered { + println!("skipped: WHP offers no nested virtualization on this host"); + return None; + } + whp_config + .set_property(whp::PartitionProperty::NestedVirtualization(true)) + .unwrap(); + } let mut exits = WHV_EXTENDED_VM_EXITS(0); - let time_abi = WhpTimeAbi::configure(&config, &mut whp_config, &mut exits).unwrap(); + let time_abi = + WhpTimeAbi::configure(&config, &mut whp_config, &mut exits, nested_virt).unwrap(); whp_config .set_property(whp::PartitionProperty::LocalApicEmulationMode( whp::abi::WHvX64LocalApicEmulationModeXApic, @@ -2554,11 +2583,24 @@ mod whp_tests { #[test] #[ignore = "requires WHP"] fn partition_meets_the_static_obligations() { + check_static_obligations(false); + } + + /// A partition with nested virtualization meets the same obligations: + /// WHP still answers the identity range past the exits, its nested + /// virtualization leaves included, with zeros, and VP 0 sees VMX. + #[test] + #[ignore = "requires WHP with nested virtualization"] + fn nested_partition_meets_the_static_obligations() { + check_static_obligations(true); + } + + fn check_static_obligations(nested_virt: bool) { let Some(Partition { partition, time_abi, .. - }) = partition(4) + }) = partition_with(4, nested_virt) else { return; }; @@ -2591,8 +2633,16 @@ mod whp_tests { .unwrap() .is_set(WHV_X64_MSR_EXIT_BITMAP::UnhandledMsrs) ); + // VMX, `CPUID.1:ECX[5]`, follows nested virtualization. + let vmx = native(&partition, 1, 0).unwrap()[2] & (1 << 5) != 0; + assert_eq!(vmx, nested_virt); + assert_eq!( + WHV_PROCESSOR_FEATURES1(time_abi.features_bank1) + .is_set(WHV_PROCESSOR_FEATURES1::NestedVirtSupport), + nested_virt + ); println!( - "tsc_hz {} bank1 {:#x} native 0x80000007 {:#x?}", + "tsc_hz {} bank1 {:#x} native 0x80000007 {:#x?} vmx {vmx}", partition.tsc_frequency().unwrap(), time_abi.features_bank1, native(&partition, 0x8000_0007, 0).unwrap() @@ -2607,13 +2657,25 @@ mod whp_tests { #[test] #[ignore = "requires WHP"] fn host_profile_configures_the_partition() { + check_host_profile_configures_the_partition(false); + } + + /// A partition with nested virtualization presents its effective CPUID, + /// VMX included, on every VP. + #[test] + #[ignore = "requires WHP with nested virtualization"] + fn host_profile_configures_a_nested_partition() { + check_host_profile_configures_the_partition(true); + } + + fn check_host_profile_configures_the_partition(nested_virt: bool) { const VP_COUNT: u32 = 4; let Some(Partition { partition, time_abi, topology, effective, - }) = partition(VP_COUNT) + }) = partition_with(VP_COUNT, nested_virt) else { return; }; From 6ac7fa50bc209b07a4c46d12169e28ce458d11ce Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 2 Nov 2026 00:15:55 -0800 Subject: [PATCH 4/4] microVM: allow nested virtualization on KVM and WHP Part of microsoft/nvx#256 and microsoft/nvx#1178. The microVM machine refused nested virtualization twice: in openvmm_defs::microvm::validate_machine_config() and in the CLI's validate_microvm_options(). With the effective CPUID of nested virtualization and WHP time ABI partitions that may have it, a microVM can give its guest Intel's VMX on KVM and WHP, so that the guest runs VMs of its own. Allow it there, as an explicit request of a cold boot: --nested-virt on the CLI and VMConfig.nested_virt in the management RPC. A microVM without the request is unchanged. Backends: KVM already recognizes nested virtualization and refuses the VM where its supported CPUID lacks VMX, as when kvm_intel's nested parameter is off; WHP refuses it where it does not offer it; and the effective CPUID refuses an AMD profile. MSHV offers nested virtualization only to a partition with the Hyper-V guest interface, which the NVX time ABI excludes, so openvmm_defs::microvm:: validate_microvm_nested_virtualization() refuses it on MSHV whenever the backend is known: in validate_machine_config(), which the worker runs with the selected backend, and in the CLI, with an explicit --hypervisor. Every refusal happens before the guest runs. Snapshots: the microVM's saved state holds no processor state of a nested guest. KVM's backend cannot save it (nested_state() returns NotSupported), MSHV refuses it, and WHP's is not part of the saved-state contract. The VMX capabilities on which a guest hypervisor's state depends are also the backend's, which no CPU profile pins. A microVM with nested virtualization is therefore never captured or restored, and each layer refuses both before anything runs: - the CLI refuses --nested-virt with --snapshot-destination or --restore-snapshot, the latter before it opens the snapshot, through the new Options::validate_microvm_nested_virtualization(), which it runs before SnapshotRestore::open() and again with the other microVM options; - the management RPC refuses nested_virt with a capture destination, and with a restore before it opens the snapshot; - the worker refuses a microVM restore that asks for nested virtualization; completes a guest's snapshot boundary request of a microVM with it at once, in filter_boundary_request(), before host input is gated or a vCPU stops, since no destination can follow; and refuses a capture request of such a microVM before anything stops, as SnapshotQuiesceError::Rejected, so that the guest continues. A snapshot thus never has nested virtualization, so the snapshot format, the manifest, and the saved-state schemas are unchanged, and a restore's recomputed effective CPUID never sets VMX. This extends microVM ABI version 3, which the layers below introduced and which no release has shipped, rather than bumping it again. Tests: - openvmm_defs: microvm_nested_virtualization_requires_kvm_or_whp accepts KVM, WHP, and an unknown backend, and refuses MSHV and every other name. - openvmm_entry: test_microvm_nested_virtualization_validation accepts --nested-virt with and without KVM or WHP, refuses it on MSHV, and refuses it with a capture or a restore; ttrpc_microvm_nested_virtualization_refuses_capture_and_restore refuses a request for both with a capture destination, and a restore that asks for it before the RPC reads its snapshot, which does not exist. - openvmm_core: snapshot_boundary_completes_a_nested_microvms_request_at_once checks that a nested microVM's boundary request ends before the worker gates input or stops a vCPU, that the guest's write completes, and that the device then takes the next request. Update the Guide's CLI, gRPC, and snapshot pages. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/reference/openvmm/management/cli.md | 29 +++-- .../src/reference/openvmm/management/grpc.md | 6 ++ Guide/src/user_guide/openvmm/snapshots.md | 13 +++ .../src/worker/dispatch/microvm.rs | 25 ++++- .../src/worker/dispatch/snapshot_rpc.rs | 45 +++++++- openvmm/openvmm_defs/src/microvm.rs | 52 ++++++++- openvmm/openvmm_entry/src/cli_args/microvm.rs | 100 +++++++++++++++++- openvmm/openvmm_entry/src/lib.rs | 3 + openvmm/openvmm_entry/src/ttrpc/microvm.rs | 80 ++++++++++++++ 9 files changed, 337 insertions(+), 16 deletions(-) diff --git a/Guide/src/reference/openvmm/management/cli.md b/Guide/src/reference/openvmm/management/cli.md index 8da4addf608..c873a9518ab 100644 --- a/Guide/src/reference/openvmm/management/cli.md +++ b/Guide/src/reference/openvmm/management/cli.md @@ -73,10 +73,10 @@ describes the source definitions. read-only; and every device but the NIC indirect descriptors and event-index notification suppression. No device offers `VIRTIO_F_ACCESS_PLATFORM`, since the microVM has no IOMMU, and no console offers its size. Firmware, ACPI, - SMBIOS, PCI, VMBus, UARTs, graphics, isolation, nested virtualization, and - other devices are rejected. Linux discovers contiguous APIC IDs, the IOAPIC, - and the routing of its 24 inputs from Intel MP 1.4 tables at `0x0` and - `0x400`; `boot_params` is at + SMBIOS, PCI, VMBus, UARTs, graphics, isolation, and other devices are + rejected. Linux discovers contiguous APIC IDs, the IOAPIC, and the routing + of its 24 inputs from Intel MP 1.4 tables at `0x0` and `0x400`; + `boot_params` is at `0x2000`, the command line starts at `0x20000`, and no ACPI or SMBIOS data is exposed. Host-driven save/restore, pulse-save/restore, and worker restart remain unavailable. @@ -91,7 +91,22 @@ describes the source definitions. with one core per vCPU, no SMT, xAPIC mode, and contiguous APIC IDs from 0. Guest-requested snapshot capture and new-process restore are available for blockless and fixed-block machines on Linux/KVM, Linux/MSHV, and Windows/WHP, - except for the RAM-backed overlay layout. + except for the RAM-backed overlay layout and nested virtualization. + + `--nested-virt` gives a `microvm` nested virtualization, so that its guest + can run VMs of its own, on KVM and WHP but never on MSHV, which offers + nested virtualization only with the Hyper-V guest interface that the NVX time + ABI excludes. The effective CPUID then sets VMX (`CPUID.1:ECX[5]`) over the + CPU profile, whose policy clears every virtualization extension, and the VMX + capability MSRs that describe the controls a guest hypervisor may use are + the backend's. It requires an Intel CPU profile, since no profile describes + SVM's feature leaf `0x8000000A` (`E_PROFILE_UNSUPPORTED` on AMD), and a + backend that offers nested virtualization on the host: KVM with `kvm_intel` + loaded with `nested=1`, or WHP with its nested virtualization feature; + otherwise the partition's creation fails before the guest runs. Such a + microVM is never captured or restored: OpenVMM refuses `--nested-virt` with + `--snapshot-destination` or `--restore-snapshot` before the VM starts. A + microVM without `--nested-virt` exposes no virtualization extension. A non-empty `--microvm-sandbox-block` topology must end with the writable `scratch` block unless the command line carries exactly one @@ -807,7 +822,9 @@ guest state and protect the directory with host access controls. such as Virtual Secure Mode (VSM), which can hurt performance and interfere with VMBus devices; nested virt cannot currently be combined with `--hv`/VMBus or - `--hypervisor whp:user_mode_apic`. + `--hypervisor whp:user_mode_apic`. A `--machine microvm` takes it on KVM and + WHP only, with an Intel CPU profile, and refuses it with a snapshot capture + or restore; see `--machine` above. * `--uefi [OPTIONS]`: Boot using `mu_msvm` UEFI. Options are comma-separated: * `firmware=`: Path to the UEFI firmware file (`MSVM.fd`). If omitted, the default is read from `OPENVMM_UEFI_FIRMWARE`, then from `X86_64_OPENVMM_UEFI_FIRMWARE` or `AARCH64_OPENVMM_UEFI_FIRMWARE`. * `debug`: Enable UEFI debugging on COM1. diff --git a/Guide/src/reference/openvmm/management/grpc.md b/Guide/src/reference/openvmm/management/grpc.md index 71b0c058761..d40723d18ac 100644 --- a/Guide/src/reference/openvmm/management/grpc.md +++ b/Guide/src/reference/openvmm/management/grpc.md @@ -267,6 +267,12 @@ value 2 offered `VIRTIO_F_ACCESS_PLATFORM` on every device but the NIC, and the console size on the consoles, and put the virtio devices on ISA IRQs, where value 3 gives each one its own IOAPIC input past them. +`VMConfig.nested_virt` gives a microVM nested virtualization, as +[`--nested-virt`](cli.md) does: on KVM and WHP, never on MSHV, and under an +Intel CPU profile, with VMX set in its effective CPUID. A request refuses it +with `microvm_snapshot.destination_path`, and a restore refuses it, since a +microVM with nested virtualization is never captured or restored. + `VMConfig.cpu_profile` selects the microVM's CPU profile as [`--cpu-profile`](cli.md) does, with the same values and checks: `auto`, the default, which an empty value also selects, for the built-in profile of the diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index 835b5c31afc..850fde3f02c 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -424,6 +424,17 @@ profile. Their error thus names every violation of each, leads with the code of the first that fails, in that order, and goes on with the failures of the others. +A microVM with nested virtualization (`--nested-virt`), whose effective CPUID +sets VMX, is never captured or restored. KVM cannot save the processor state +of a nested guest, and the VMX capabilities on which a guest hypervisor's +state depends are the backend's, which no CPU profile pins. OpenVMM therefore +refuses `--nested-virt` with `--snapshot-destination` or +`--restore-snapshot`, and the management RPC refuses `nested_virt` with +`destination_path` or `restore_path`, before the VM starts; a guest's +snapshot request in such a VM is also refused before anything stops. No +snapshot records VMX, so a restore, which computes the effective CPUID +without it, never has nested virtualization. + Restore also requires: - a snapshot of the same time ABI version, so a snapshot of time ABI 1 must @@ -614,3 +625,5 @@ immediately with a clear error if any active device does not support it. model and stepping whose backend supports the profile. Snapshots from manifest versions before 6, or from a build that computes a different effective CPUID, must be recaptured. +- A microVM with nested virtualization (`--nested-virt`) supports neither + capture nor restore. diff --git a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs index 7cee87dd756..6b11c2481ed 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs @@ -106,6 +106,14 @@ impl MicrovmParameters { ¶meters.cfg, Some(parameters.hypervisor.id()), )?; + // A snapshot never records nested virtualization, since no microVM + // with it is captured, so its restore never has it either. + anyhow::ensure!( + parameters.cfg.machine_profile != MachineProfile::Microvm + || !parameters.cfg.hypervisor.nested_virt + || parameters.saved_state.is_none(), + "a microVM restore never has nested virtualization" + ); Ok(Self { snapshot_boundary }) } @@ -197,7 +205,9 @@ impl LoadedVm { &mut self, request: MicrovmSnapshotBoundaryRequest, ) -> bool { - let Some(request) = snapshot_rpc::filter_boundary_request(request) else { + let Some(request) = + snapshot_rpc::filter_boundary_request(request, self.inner.hypervisor_cfg.nested_virt) + else { return true; }; if self.snapshot_boundary.stop_guard.is_some() { @@ -327,6 +337,19 @@ impl LoadedVm { RemoteError::new(anyhow::anyhow!("VM is already stopped")), )); } + // A microVM with nested virtualization is never captured: KVM cannot + // save a nested guest's processor state, and the VMX capabilities on + // which a guest hypervisor's state depends are the host backend's, + // which no CPU profile pins. The frontends refuse such a capture at + // launch; refuse it here too, before anything stops, rather than + // during the save. + if self.inner.hypervisor_cfg.nested_virt { + return Err(openvmm_defs::rpc::SnapshotQuiesceError::Rejected( + RemoteError::new(anyhow::anyhow!( + "microVM snapshot capture does not support nested virtualization" + )), + )); + } let quiesce = openvmm_defs::profile::ProfileSpan::start(); if let Err(error) = self.state_units.quiesce_for_save(timeout).await { diff --git a/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs b/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs index 8bf211c05c0..a4c90f6b0f5 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs @@ -12,8 +12,14 @@ use openvmm_defs::rpc::VmRpc; #[error("VM mutation is unavailable while a microVM snapshot boundary is active")] struct SnapshotBoundaryActive; +/// Returns the guest's snapshot boundary request that the worker must +/// establish, or `None` for one that ends before host input is gated or a vCPU +/// stops: one that stop or reset cancelled, and any of a microVM with nested +/// virtualization (`nested_virt`), which is never captured, which completes at +/// once. pub(super) fn filter_boundary_request( request: MicrovmSnapshotBoundaryRequest, + nested_virt: bool, ) -> Option { // Stop/reset drops the release receiver but leaves the request queued. // The worker serializes reset with boundary establishment, so checking @@ -22,6 +28,14 @@ pub(super) fn filter_boundary_request( tracelimit::warn_ratelimited!("dropping cancelled microVM snapshot boundary request"); return None; } + if nested_virt { + tracelimit::warn_ratelimited!( + "refusing microVM snapshot boundary request: a microVM with nested virtualization is never captured" + ); + request.release_write.send(()); + request.transaction_complete.complete(()); + return None; + } Some(request) } @@ -113,9 +127,9 @@ mod tests { // Cancellation leaves the old request ahead of the new one in the queue. let old_request = requests.try_recv().unwrap(); assert!(old_request.release_write.is_closed()); - assert!(filter_boundary_request(old_request).is_none()); + assert!(filter_boundary_request(old_request, false).is_none()); - let request = filter_boundary_request(requests.try_recv().unwrap()) + let request = filter_boundary_request(requests.try_recv().unwrap(), false) .expect("a cancelled predecessor must not discard the next request"); request.release_write.send(()); device.poll_device(&mut cx); @@ -137,7 +151,7 @@ mod tests { let IoResult::Defer(mut write) = device.io_write(SNAPSHOT_PORT, &[0]) else { panic!("snapshot write was not deferred"); }; - let request = filter_boundary_request(requests.try_recv().unwrap()) + let request = filter_boundary_request(requests.try_recv().unwrap(), false) .expect("live snapshot request was discarded"); request.release_write.send(()); let mut cx = Context::from_waker(Waker::noop()); @@ -156,7 +170,30 @@ mod tests { IoResult::Defer(_) )); drop(device); - assert!(filter_boundary_request(requests.try_recv().unwrap()).is_none()); + assert!(filter_boundary_request(requests.try_recv().unwrap(), false).is_none()); + } + + #[test] + fn snapshot_boundary_completes_a_nested_microvms_request_at_once() { + let (send, mut requests) = mesh::channel(); + let mut device = MicrovmSnapshotRequest::new(Some(send), Duration::from_secs(1)); + device.start(); + let IoResult::Defer(mut write) = device.io_write(SNAPSHOT_PORT, &[0]) else { + panic!("snapshot write was not deferred"); + }; + // The request ends before the worker gates input or stops a vCPU, + // and the guest's write completes. + assert!(filter_boundary_request(requests.try_recv().unwrap(), true).is_none()); + let mut cx = Context::from_waker(Waker::noop()); + device.poll_device(&mut cx); + assert!(matches!(write.poll_write(&mut cx), Poll::Ready(Ok(())))); + + // The device takes the next request, which the transaction's + // completion released. + let IoResult::Defer(_) = device.io_write(SNAPSHOT_PORT, &[0]) else { + panic!("the next snapshot write was not deferred"); + }; + assert!(filter_boundary_request(requests.try_recv().unwrap(), true).is_none()); } #[test] diff --git a/openvmm/openvmm_defs/src/microvm.rs b/openvmm/openvmm_defs/src/microvm.rs index e18afed1266..e7d08ea5490 100644 --- a/openvmm/openvmm_defs/src/microvm.rs +++ b/openvmm/openvmm_defs/src/microvm.rs @@ -1950,6 +1950,30 @@ fn validate_machine_load_mode( } } +/// The backends whose microVMs may have nested virtualization. +pub const MICROVM_NESTED_VIRTUALIZATION_BACKENDS: [&str; 2] = ["kvm", "whp"]; + +/// Checks that a microVM with nested virtualization runs on a backend that +/// supports it, KVM or WHP, when `hypervisor_id` names the backend. MSHV +/// exposes nested virtualization only to a partition with the Hyper-V guest +/// interface, which the NVX time ABI excludes, so a microVM on MSHV never has +/// it. +/// +/// Whether the host itself offers nested virtualization, and whether the +/// microVM's CPU profile is Intel's, whose VMX nested virtualization exposes, +/// is checked when the partition is created, before the guest runs. +pub fn validate_microvm_nested_virtualization(hypervisor_id: Option<&str>) -> anyhow::Result<()> { + if let Some(hypervisor_id) = hypervisor_id { + anyhow::ensure!( + MICROVM_NESTED_VIRTUALIZATION_BACKENDS.contains(&hypervisor_id), + "microVM nested virtualization requires KVM or WHP, not {hypervisor_id}: MSHV \ + exposes nested virtualization only with the Hyper-V guest interface, which the \ + NVX time ABI excludes" + ); + } + Ok(()) +} + /// Validates the microVM machine contract. Standard-machine configurations are unchanged. pub fn validate_machine_config(config: &Config, hypervisor_id: Option<&str>) -> anyhow::Result<()> { let MachineProfile::Microvm = config.machine_profile else { @@ -2036,10 +2060,12 @@ pub fn validate_machine_config(config: &Config, hypervisor_id: Option<&str>) -> anyhow::ensure!( !config.hypervisor.with_hv && config.hypervisor.with_vtl2.is_none() - && config.hypervisor.with_isolation.is_none() - && !config.hypervisor.nested_virt, - "microVM does not support Hyper-V enlightenments, VTL2, isolation, or nested virtualization" + && config.hypervisor.with_isolation.is_none(), + "microVM does not support Hyper-V enlightenments, VTL2, or isolation" ); + if config.hypervisor.nested_virt { + validate_microvm_nested_virtualization(hypervisor_id)?; + } let expected_chipset = BaseChipsetManifest { with_generic_cmos_rtc: true, @@ -2332,6 +2358,26 @@ mod tests { ); } + /// A microVM may have nested virtualization on KVM and WHP, and on a + /// backend that is not yet selected, but never on MSHV. + #[test] + fn microvm_nested_virtualization_requires_kvm_or_whp() { + for hypervisor_id in [None, Some("kvm"), Some("whp")] { + validate_microvm_nested_virtualization(hypervisor_id).unwrap(); + } + for hypervisor_id in ["mshv", "hvf", ""] { + let error = validate_microvm_nested_virtualization(Some(hypervisor_id)) + .unwrap_err() + .to_string(); + assert!( + error.starts_with(&format!( + "microVM nested virtualization requires KVM or WHP, not {hypervisor_id}:" + )), + "{error}" + ); + } + } + #[test] fn microvm_processor_limit_is_host_owned() { let mut cmdline = build_microvm_command_line(&[], false).unwrap(); diff --git a/openvmm/openvmm_entry/src/cli_args/microvm.rs b/openvmm/openvmm_entry/src/cli_args/microvm.rs index 6ca3490d47d..1ae8456ca9d 100644 --- a/openvmm/openvmm_entry/src/cli_args/microvm.rs +++ b/openvmm/openvmm_entry/src/cli_args/microvm.rs @@ -783,6 +783,34 @@ impl Options { && (self.restore_snapshot.is_none() || self.microvm.restore_network_policy) } + /// Checks `--nested-virt` of a microVM: never on MSHV, when + /// `--hypervisor` names the backend, and never with a capture or a + /// restore. No backend saves a nested guest's processor state for the + /// microVM, so a machine with nested virtualization is neither captured + /// nor restored, and a snapshot never has it: refuse both before the VM + /// starts rather than during the state transfer. OpenVMM runs this before + /// it reads a snapshot, and again with the other microVM options. + pub(crate) fn validate_microvm_nested_virtualization(&self) -> anyhow::Result<()> { + if self.machine != MachineProfileCli::Microvm || !self.nested_virt { + return Ok(()); + } + if let Some(hypervisor) = self.hypervisor.as_deref() { + let name = hypervisor.split(':').next().unwrap_or(hypervisor); + openvmm_defs::microvm::validate_microvm_nested_virtualization(Some(name))?; + } + anyhow::ensure!( + self.microvm.snapshot_destination.is_none(), + "microVM nested virtualization does not support snapshot capture; omit \ + --snapshot-destination or --nested-virt" + ); + anyhow::ensure!( + self.restore_snapshot.is_none(), + "a microVM restore takes its machine from the snapshot, which never has nested \ + virtualization; omit --nested-virt" + ); + Ok(()) + } + pub(crate) fn validate_microvm_options(&self) -> anyhow::Result<()> { self.validate_microvm_host_loopback()?; if self.machine != MachineProfileCli::Microvm { @@ -996,14 +1024,13 @@ impl Options { !self.hv && !self.vtl2 && self.isolation.is_none() - && !self.nested_virt && !self.get && !self.vmbus_redirect && self.vmbus_vsock_path.is_none() && self.vmbus_vtl2_vsock_path.is_none() && self.openhcl_dump_path.is_none() && self.gdb.is_none(), - "microVM does not support Hyper-V, VTL2, isolation, nested virtualization, GET, or VMBus" + "microVM does not support Hyper-V, VTL2, isolation, GET, or VMBus" ); if let Some(hypervisor) = self.hypervisor.as_deref() { let name = hypervisor.split(':').next().unwrap_or(hypervisor); @@ -1013,6 +1040,7 @@ impl Options { "microVM requires KVM or MSHV on Linux, or WHP on Windows" ); } + self.validate_microvm_nested_virtualization()?; anyhow::ensure!( self.com1.is_none() @@ -1584,6 +1612,74 @@ mod tests { } } + /// `--nested-virt` gives a microVM nested virtualization on KVM and WHP, + /// never on MSHV, and never with a capture or a restore, which the + /// options refuse before the VM starts, and before OpenVMM reads the + /// snapshot of a restore. + #[cfg(guest_arch = "x86_64")] + #[test] + fn test_microvm_nested_virtualization_validation() { + let microvm = |extra: &[&str]| { + let mut args = vec!["openvmm", "--machine", "microvm"]; + args.extend_from_slice(extra); + Options::try_parse_from(args).unwrap() + }; + assert!(!microvm(&[]).nested_virt); + microvm(&["--nested-virt"]) + .validate_microvm_options() + .unwrap(); + let backend = if cfg!(windows) { "whp" } else { "kvm" }; + let nested = microvm(&["--nested-virt", "--hypervisor", backend]); + assert!(nested.nested_virt); + nested.validate_microvm_options().unwrap(); + + #[cfg(target_os = "linux")] + { + let error = microvm(&["--nested-virt", "--hypervisor", "mshv"]) + .validate_microvm_options() + .unwrap_err() + .to_string(); + assert!( + error.starts_with("microVM nested virtualization requires KVM or WHP, not mshv:"), + "{error}" + ); + microvm(&["--hypervisor", "mshv"]) + .validate_microvm_options() + .unwrap(); + } + + let capture = microvm(&["--nested-virt", "--snapshot-destination", "snapshot"]); + assert_eq!( + capture.validate_microvm_options().unwrap_err().to_string(), + "microVM nested virtualization does not support snapshot capture; omit \ + --snapshot-destination or --nested-virt" + ); + microvm(&["--snapshot-destination", "snapshot"]) + .validate_microvm_options() + .unwrap(); + let restore = microvm(&["--nested-virt", "--restore-snapshot", "snapshot"]); + for error in [ + restore.validate_microvm_options().unwrap_err(), + restore + .validate_microvm_nested_virtualization() + .unwrap_err(), + ] { + assert_eq!( + error.to_string(), + "a microVM restore takes its machine from the snapshot, which never has nested \ + virtualization; omit --nested-virt" + ); + } + microvm(&["--restore-snapshot", "snapshot"]) + .validate_microvm_nested_virtualization() + .unwrap(); + // Only the microVM machine checks the request here. + Options::try_parse_from(["openvmm", "--nested-virt", "--restore-snapshot", "snapshot"]) + .unwrap() + .validate_microvm_nested_virtualization() + .unwrap(); + } + #[cfg(guest_arch = "x86_64")] #[test] fn test_microvm_memory_capacity_and_restore_target_parsing() { diff --git a/openvmm/openvmm_entry/src/lib.rs b/openvmm/openvmm_entry/src/lib.rs index c3827ab0e48..715f0531f84 100644 --- a/openvmm/openvmm_entry/src/lib.rs +++ b/openvmm/openvmm_entry/src/lib.rs @@ -2889,6 +2889,9 @@ async fn run_control_inner( mut opt: Options, ) -> anyhow::Result { let mesh = mesh_slot.as_ref().unwrap(); + // A snapshot never has nested virtualization, so refuse a restore that + // asks for it before reading the snapshot. + opt.validate_microvm_nested_virtualization()?; let mut restore = snapshot_restore::SnapshotRestore::open(&opt)?; let microvm_restore = microvm::prepare_restore(&mut opt, restore.snapshot())?; let (mut vm_config, mut resources) = diff --git a/openvmm/openvmm_entry/src/ttrpc/microvm.rs b/openvmm/openvmm_entry/src/ttrpc/microvm.rs index 700ca718e00..207cb400d4a 100644 --- a/openvmm/openvmm_entry/src/ttrpc/microvm.rs +++ b/openvmm/openvmm_entry/src/ttrpc/microvm.rs @@ -153,6 +153,17 @@ impl CreateVm { snapshot_destination.is_none() || restore_path.is_none(), "snapshot capture and restore paths are mutually exclusive" ); + // A restore takes its machine from the snapshot, which never has + // nested virtualization, since no microVM with it is captured: refuse + // the request before the snapshot is read. + anyhow::ensure!( + restore_path.is_none() + || !requested_config + .as_ref() + .is_some_and(|config| config.nested_virt), + "a microVM restore takes its machine from the snapshot, which never has nested \ + virtualization; clear nested_virt" + ); anyhow::ensure!( !restore_entropy || restore_path.is_some(), "restore_entropy requires restore_path" @@ -329,6 +340,14 @@ impl CreateVm { active || (snapshot_destination.is_none() && authoritative_restore.is_none()), "microVM snapshot options require the microVM machine profile" ); + // No backend saves a nested guest's processor state for the microVM, + // so refuse its capture before the VM starts rather than during the + // state transfer. + anyhow::ensure!( + !(active && req_config.nested_virt && snapshot_destination.is_some()), + "microVM nested virtualization does not support snapshot capture; clear nested_virt \ + or destination_path" + ); anyhow::ensure!( active || cpu_profile.is_empty(), "cpu_profile requires the microVM machine profile" @@ -1665,6 +1684,67 @@ mod tests { } } + /// A microVM over the management RPC may have nested virtualization, but + /// not with a capture or a restore, which the request refuses before the + /// VM starts, and, for a restore, before it reads the snapshot. + #[test] + fn ttrpc_microvm_nested_virtualization_refuses_capture_and_restore() { + let request = |nested_virt: bool, destination: &str| vmservice::CreateVmRequest { + config: Some(vmservice::VmConfig { + machine_profile: vmservice::vm_config::MachineProfile::Microvm as i32, + nested_virt, + ..Default::default() + }), + microvm_snapshot: Some(vmservice::MicrovmSnapshotConfig { + destination_path: destination.to_owned(), + ..Default::default() + }), + ..Default::default() + }; + let restore = |nested_virt: bool| vmservice::CreateVmRequest { + config: Some(vmservice::VmConfig { + machine_profile: vmservice::vm_config::MachineProfile::Microvm as i32, + nested_virt, + ..Default::default() + }), + microvm_snapshot: Some(vmservice::MicrovmSnapshotConfig { + restore_path: "missing-snapshot".to_owned(), + ..Default::default() + }), + ..Default::default() + }; + let Err(error) = CreateVm::new(request(true, "snapshot")) else { + panic!("a nested microVM must refuse a capture"); + }; + assert_eq!( + error.to_string(), + "microVM nested virtualization does not support snapshot capture; clear nested_virt \ + or destination_path" + ); + let (_, config) = CreateVm::new(request(true, "")).unwrap(); + assert!(config.nested_virt); + let (create, config) = CreateVm::new(request(false, "snapshot")).unwrap(); + assert!(!config.nested_virt); + assert!(create.snapshot_destination.is_some()); + // The restore's refusal precedes its read of the snapshot, which does + // not exist, and which a restore without the request fails to open. + let Err(error) = CreateVm::new(restore(true)) else { + panic!("a restore must refuse nested virtualization"); + }; + assert_eq!( + error.to_string(), + "a microVM restore takes its machine from the snapshot, which never has nested \ + virtualization; clear nested_virt" + ); + let Err(error) = CreateVm::new(restore(false)) else { + panic!("a restore of a missing snapshot must fail"); + }; + assert!( + format!("{error:#}").contains("failed to open snapshot directory"), + "{error:#}" + ); + } + #[test] fn ttrpc_cpu_profile_requires_the_microvm_profile() { let Err(error) = CreateVm::new(vmservice::CreateVmRequest {