From ce8ea0c057cff575d4e31363e00a726f747c3bc0 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 6 Nov 2026 18:19:31 -0800 Subject: [PATCH] virt_kvm: clear a pending NMI and the interrupt shadow on reset and restore Setting a VP's activity on KVM left a pending NMI and an interrupt shadow in place when the new activity had them clear. KVM_SET_VCPU_EVENTS changes the pending NMI only with KVM_VCPUEVENT_VALID_NMI_PENDING and the interrupt shadow only with KVM_VCPUEVENT_VALID_SHADOW, and KvmVpStateAccess::set_activity passed each flag only for a true value, so a false value kept KVM's state instead of replacing it. A VP reset, which sets the activity at reset, therefore kept an NMI that the VP had not taken and the shadow of an STI or MOV SS: the rebooted guest could take the stale NMI as soon as the VP started, before it could handle one, and a debug build failed reset_all's check with a reset state mismatch. Restoring an activity with either value clear onto a VP that had it set kept the VP's value as well, which a debug build reports as a restore state mismatch. microsoft/openvmm#4014 passed the flags only for true values so as not to discard an NMI that another VP sends between reading and setting the activity. On KVM, though, only a reset and a restore set the activity, and both run while the VPs are stopped; KVM handles INIT and SIPI itself, and no other VP can change a VP's interrupt shadow. Always pass both flags, as the MSHV and WHP backends already replace both values. Setting true values behaves as before, KVM_SET_VCPU_EVENTS sets NMI masking and the injected events whatever the flags, and OpenVMM passes no other flag, so it still leaves the SMM state and the SIPI vector alone. The run loop's ExtINT injection, which does run while other VPs run, still passes no flags. KVM still cannot withdraw an NMI that it has handed to AMD's virtual NMI, the VMCB's V_NMI_PENDING, which it uses when the kvm_amd vnmi parameter is Y: KVM_SET_VCPU_EVENTS clears only its own count of pending NMIs. The new tests need /dev/kvm and are ignored by default. They check that resetting a BSP and an AP clears an NMI sent by MSI and an interrupt shadow, and every transition of the pending NMI, NMI masking, and the interrupt shadow through set_activity. On a host that uses AMD's virtual NMI, they skip the pending NMI. Reported in microsoft/nvx#1159. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Cargo.lock | 1 + vmm_core/virt_kvm/Cargo.toml | 1 + vmm_core/virt_kvm/src/arch/x86_64/vp_state.rs | 196 +++++++++++++++++- 3 files changed, 191 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2c2dfb39b9..9d9ddcdf055 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10417,6 +10417,7 @@ dependencies = [ "libc", "memory_range", "open_enum", + "pal_async", "pal_event", "parking_lot", "pci_core", diff --git a/vmm_core/virt_kvm/Cargo.toml b/vmm_core/virt_kvm/Cargo.toml index 54a6cd21188..78c61067556 100644 --- a/vmm_core/virt_kvm/Cargo.toml +++ b/vmm_core/virt_kvm/Cargo.toml @@ -34,6 +34,7 @@ tracelimit.workspace = true zerocopy.workspace = true [dev-dependencies] +pal_async.workspace = true test_with_tracing.workspace = true [target.'cfg(all(target_os = "linux", target_arch = "aarch64"))'.dependencies] diff --git a/vmm_core/virt_kvm/src/arch/x86_64/vp_state.rs b/vmm_core/virt_kvm/src/arch/x86_64/vp_state.rs index 59dd45bac5d..65e3856a112 100644 --- a/vmm_core/virt_kvm/src/arch/x86_64/vp_state.rs +++ b/vmm_core/virt_kvm/src/arch/x86_64/vp_state.rs @@ -353,13 +353,12 @@ impl AccessVpState for KvmVpStateAccess<'_, '_> { }; self.kvm().set_mp_state(state)?; - let mut event_flags = 0; - if nmi_pending { - event_flags |= kvm::KVM_VCPUEVENT_VALID_NMI_PENDING; - } - if interrupt_shadow { - event_flags |= kvm::KVM_VCPUEVENT_VALID_SHADOW; - } + // KVM changes the pending NMI and the interrupt shadow only when these + // flags are set, so set them for false values too: a reset or a + // restore must clear them. Both happen while the VPs are stopped, so + // no other VP can send an NMI that this would discard. (KVM cannot + // withdraw an NMI that it has handed to AMD's virtual NMI, though.) + let event_flags = kvm::KVM_VCPUEVENT_VALID_NMI_PENDING | kvm::KVM_VCPUEVENT_VALID_SHADOW; let mut events = kvm::kvm_vcpu_events { exception: kvm::kvm_vcpu_events__bindgen_ty_1 { @@ -708,3 +707,186 @@ impl AccessVpState for KvmVpStateAccess<'_, '_> { Err(KvmError::NotSupported) } } + +/// Tests against the host's KVM. +#[cfg(test)] +mod tests { + use crate::Kvm; + use crate::KvmPartition; + use crate::KvmProcessorBinder; + use guestmem::GuestMemory; + use hvdef::Vtl; + use pal_async::DefaultDriver; + use pal_async::async_test; + use test_with_tracing::test; + use virt::BindProcessor; + use virt::Hypervisor; + use virt::Partition; + use virt::PartitionConfig; + use virt::Processor; + use virt::ProtoPartition; + use virt::ProtoPartitionConfig; + use virt::irqcon::DeliveryMode; + use virt::irqcon::MsiRequest; + use virt::state::StateElement; + use virt::x86::vp; + use virt::x86::vp::AccessVpState; + use vm_topology::memory::MemoryLayout; + use vm_topology::processor::ProcessorTopology; + use vm_topology::processor::TopologyBuilder; + use vmcore::vmtime::VmTime; + use vmcore::vmtime::VmTimeKeeper; + + /// Returns whether KVM hands pending NMIs to AMD's virtual NMI (the VMCB's + /// V_NMI_PENDING), from which `KVM_SET_VCPU_EVENTS` cannot withdraw them. + fn kvm_uses_vnmi() -> bool { + std::fs::read_to_string("/sys/module/kvm_amd/parameters/vnmi") + .is_ok_and(|value| value.trim() == "Y") + } + + /// A partition on the host's KVM whose VPs never run, so that it maps no + /// RAM. + struct Fixture { + topology: ProcessorTopology, + partition: KvmPartition, + binders: Vec, + _keeper: VmTimeKeeper, + } + + impl Fixture { + async fn new(driver: &DefaultDriver, vp_count: u32) -> Self { + let keeper = VmTimeKeeper::new(driver, VmTime::from_100ns(0)); + let vmtime = keeper.builder().build(driver).await.unwrap(); + let topology = TopologyBuilder::new_x86().build(vp_count).unwrap(); + let mem_layout = MemoryLayout::new(4096, &[], &[], &[], None).unwrap(); + let guest_memory = GuestMemory::allocate(4096); + let (partition, binders) = Kvm::new() + .unwrap() + .new_partition(ProtoPartitionConfig { + processor_topology: &topology, + hv_config: None, + vmtime: &vmtime, + isolation: virt::ProtoPartitionIsolation::None, + nested_virt: false, + }) + .unwrap() + .build(PartitionConfig { + mem_layout: &mem_layout, + guest_memory: &guest_memory, + cpuid: &[], + vtl0_alias_map: None, + fault_resolver: None, + }) + .unwrap(); + Self { + topology, + partition, + binders, + _keeper: keeper, + } + } + } + + /// Checks that the reset of a VP, the VP part of a VM reset, discards an + /// NMI that the VP has not taken and an interrupt shadow (after STI or MOV + /// SS), on the BSP and on an AP. + #[async_test] + #[ignore = "requires access to /dev/kvm"] + async fn reset_clears_a_pending_nmi_and_the_interrupt_shadow(driver: DefaultDriver) { + // A pending NMI that KVM has handed to AMD's virtual NMI survives a + // reset, so only check the interrupt shadow there. + let nmi = !kvm_uses_vnmi(); + if !nmi { + tracing::info!("KVM uses AMD's virtual NMI: not checking a pending NMI"); + } + let mut fixture = Fixture::new(&driver, 2).await; + let Fixture { + topology, + partition, + binders, + .. + } = &mut fixture; + let mut vps = binders + .iter_mut() + .map(|binder| binder.bind().unwrap()) + .collect::>(); + + for (vp, vp_info) in vps.iter_mut().zip(topology.vps_arch()) { + if nmi { + partition.request_msi( + Vtl::Vtl0, + MsiRequest::new_x86(DeliveryMode::NMI, vp_info.apic_id, false, 0, false), + ); + } + let mut access = vp.access_state(Vtl::Vtl0); + let mut activity = access.activity().unwrap(); + assert_eq!(activity.nmi_pending, nmi); + activity.interrupt_shadow = true; + access.set_activity(&activity).unwrap(); + let activity = access.activity().unwrap(); + assert_eq!( + (activity.nmi_pending, activity.interrupt_shadow), + (nmi, true) + ); + } + + partition.supports_reset().unwrap().reset().unwrap(); + for (vp, vp_info) in vps.iter_mut().zip(topology.vps_arch()) { + vp.reset().unwrap(); + assert_eq!( + vp.access_state(Vtl::Vtl0).activity().unwrap(), + vp::Activity::at_reset(partition.caps(), &vp_info) + ); + } + } + + /// Checks that setting the activity, as a restore does, replaces the + /// pending NMI, NMI masking, and the interrupt shadow, including clearing + /// them. + #[async_test] + #[ignore = "requires access to /dev/kvm"] + async fn set_activity_replaces_the_pending_nmi_and_the_interrupt_shadow( + driver: DefaultDriver, + ) { + // KVM cannot withdraw a pending NMI that it has handed to AMD's virtual + // NMI, so only check the other state there. + let nmi_values: &[bool] = if kvm_uses_vnmi() { + tracing::info!("KVM uses AMD's virtual NMI: not checking a pending NMI"); + &[false] + } else { + &[false, true] + }; + let mut values = Vec::new(); + for &nmi_pending in nmi_values { + for nmi_masked in [false, true] { + for interrupt_shadow in [false, true] { + values.push((nmi_pending, nmi_masked, interrupt_shadow)); + } + } + } + + let mut fixture = Fixture::new(&driver, 1).await; + let mut vp = fixture.binders[0].bind().unwrap(); + let mut access = vp.access_state(Vtl::Vtl0); + for &from in &values { + for &to in &values { + for (nmi_pending, nmi_masked, interrupt_shadow) in [from, to] { + let activity = vp::Activity { + mp_state: vp::MpState::Running, + nmi_pending, + nmi_masked, + interrupt_shadow, + pending_event: None, + pending_interruption: None, + }; + access.set_activity(&activity).unwrap(); + assert_eq!( + access.activity().unwrap(), + activity, + "{from:?} -> {to:?}" + ); + } + } + } + } +}