Skip to content

openvmm: reserved REMOTE_READ delivery exposes an unsaved KVM CPU wakeup #1171

Closed

Description

@tangruize

A guest IO-APIC/MSI delivery using mode 3 reaches KVM's paravirtual CPU-kick path and sets pv_unhalted, a wakeup latch absent from OpenVMM's saved CPU state. Capture and restore can consequently disagree about whether the CPU should wake.

In vmm_core/virt_kvm/src/arch/x86_64/mod.rs, KvmMsi::new copies the device's delivery mode into the KVM message:

let data = MsiData::new()
    .with_delivery_mode(request_data.delivery_mode())
    .with_assert(request_data.assert())
    .with_destination_mode_logical(request_data.destination_mode_logical())
    .with_trigger_mode_level(request_data.trigger_mode_level())
    .with_vector(request_data.vector())
    .into();

The converter validates the MSI address but does not reject REMOTE_READ (011). That encoding is reserved for MSI and IO-APIC device delivery, whereas KVM interprets it as its private PV kick.

Trigger

Program an IO-APIC redirection entry with delivery mode 3 and assert its line during snapshot capture; paravirtual CPUID exposure is unnecessary.

For a halted CPU, the kick can make KVM_GET_MP_STATE report runnable activity. For a running CPU, the latch can remain invisible in saved activity but make its next HLT return immediately. Restoring the snapshot loses the latch, so the restored CPU instead halts.

Fix

fix.patch rejects mode 3 in KvmMsi::new, before direct MSI delivery or IO-APIC/irqfd route construction reaches KVM. Invalid encodings use the existing rate-limited logging.

The other delivery modes retain their destination, vector and edge/level semantics. The fix rejects a reserved device encoding rather than adding KVM's private wakeup latch to the snapshot format.

Activity

  1. tangruize commented on Nov 1, 2026

    @tangruize
    ContributorAuthor

    Proposed patch

    Patch against OpenVMM 998b9f2402050cd16d1faa897bc831ef1434be48. Save the contents below as fix.patch and run git apply fix.patch.

    fix.patch
    diff --git a/Cargo.lock b/Cargo.lock
    index 55754c811..e67618ebc 100644
    --- a/Cargo.lock
    +++ b/Cargo.lock
    @@ -10239,6 +10239,7 @@ dependencies = [
      "libc",
      "memory_range",
      "open_enum",
    + "pal_async",
      "pal_event",
      "parking_lot",
      "pci_core",
    diff --git a/vmm_core/virt_kvm/Cargo.toml b/vmm_core/virt_kvm/Cargo.toml
    index 28d2745f7..c765b5296 100644
    --- a/vmm_core/virt_kvm/Cargo.toml
    +++ b/vmm_core/virt_kvm/Cargo.toml
    @@ -36,6 +36,10 @@ tracelimit.workspace = true
     zerocopy.workspace = true
     
     [dev-dependencies]
    +vm_topology.workspace = true
    +vmcore.workspace = true
    +pal_async.workspace = true
    +guestmem.workspace = true
     test_with_tracing.workspace = true
     
     [target.'cfg(all(target_os = "linux", target_arch = "aarch64"))'.dependencies]
    diff --git a/vmm_core/virt_kvm/src/arch/x86_64/mod.rs b/vmm_core/virt_kvm/src/arch/x86_64/mod.rs
    index d6b9daafa..88a583949 100644
    --- a/vmm_core/virt_kvm/src/arch/x86_64/mod.rs
    +++ b/vmm_core/virt_kvm/src/arch/x86_64/mod.rs
    @@ -12,6 +12,8 @@
     pub(crate) mod snp;
     mod time_abi;
     mod vm_state;
    +#[cfg(test)]
    +mod remote_read_tests;
     mod vp_state;
     
     pub(crate) use time_abi::KvmTimeAbi;
    @@ -1361,6 +1363,17 @@ pub(crate) fn new(request: MsiRequest) -> Option<Self> {
                 return None;
             }
             let request_data = MsiData::from(request.data);
    +        // Delivery mode 011 is reserved, in the MSI data register and in an
    +        // IO-APIC redirection entry alike (Intel SDM Vol. 3A, the MSI
    +        // "Message Data Register Format"; the I/O APIC's redirection table).
    +        // KVM takes it as its PV kick (APIC_DM_REMRD, which kvm_pv_kick_cpu_op
    +        // sends: Linux v6.17 x86.c 9841-9850): it marks the target vCPU
    +        // unhalted (lapic.c 1304-1309), so KVM_GET_MP_STATE reports a halted
    +        // vCPU runnable (x86.c 11775-11778), a stopped VP's too (BUG-21). Drop
    +        // it, as an invalid encoding.
    +        if DeliveryMode(request_data.delivery_mode()) == DeliveryMode::REMOTE_READ {
    +            return None;
    +        }
     
             // Although architecturally the destination mode bit is only supposed to
             // be considered when the redirection hint bit is set, KVM always gets
    @@ -1401,7 +1414,7 @@ fn request_msi(&self, request: MsiRequest) {
                 tracelimit::warn_ratelimited!(
                     address = request.address,
                     data = request.data,
    -                "invalid MSI address"
    +                "invalid MSI address or delivery mode"
                 );
                 return;
             };
    @@ -1472,7 +1485,7 @@ fn set_irq_route(&self, irq: u8, request: Option<MsiRequest>) {
                             irq,
                             address = request.address,
                             data = request.data,
    -                        "invalid MSI address for IO-APIC route"
    +                        "invalid MSI address or delivery mode for IO-APIC route"
                         );
                         None
                     }
    diff --git a/vmm_core/virt_kvm/src/arch/x86_64/remote_read_tests.rs b/vmm_core/virt_kvm/src/arch/x86_64/remote_read_tests.rs
    new file mode 100644
    index 000000000..10d565726
    --- /dev/null
    +++ b/vmm_core/virt_kvm/src/arch/x86_64/remote_read_tests.rs
    @@ -0,0 +1,135 @@
    +// Copyright (c) Microsoft Corporation.
    +// Licensed under the MIT License.
    +
    +use super::*;
    +use guestmem::GuestMemory;
    +use pal_async::DefaultPool;
    +use test_with_tracing::test;
    +use virt::BindProcessor;
    +use virt::Hypervisor;
    +use virt::PartitionConfig;
    +use virt::ProtoPartitionConfig;
    +use virt::x86::vp;
    +use vm_topology::memory::MemoryLayout;
    +use vm_topology::processor::TopologyBuilder;
    +use vmcore::vmtime::VmTime;
    +use vmcore::vmtime::VmTimeKeeper;
    +use x86defs::apic::DeliveryMode;
    +
    +/// Builds a KVM partition with `vp_count` VPs (no guest memory mapped: the
    +/// VPs never run) and calls `f` with it and its VP binders.
    +fn with_partition<R>(
    +    vp_count: u32,
    +    f: impl FnOnce(&KvmPartition, Vec<KvmProcessorBinder>) -> R,
    +) -> R {
    +    DefaultPool::run_with(async |driver| {
    +        let keeper = VmTimeKeeper::new(&driver, VmTime::from_100ns(0));
    +        let vmtime = keeper.builder().build(&driver).await.unwrap();
    +        let topology = TopologyBuilder::new_x86().build(vp_count).unwrap();
    +        let mem_layout = MemoryLayout::new(0x400000, &[], &[], &[], None).unwrap();
    +        let guest_memory = GuestMemory::allocate(mem_layout.end_of_ram() as usize);
    +        let (partition, binders) = Kvm::new()
    +            .unwrap()
    +            .new_partition(ProtoPartitionConfig {
    +                processor_topology: &topology,
    +                hv_config: None,
    +                vmtime: &vmtime,
    +                isolation: virt::ProtoPartitionIsolation::None,
    +                nested_virt: false,
    +                user_mode_memory_faults: false,
    +                lazy_memory_registration: false,
    +                time_abi: None,
    +            })
    +            .unwrap()
    +            .build(PartitionConfig {
    +                mem_layout: &mem_layout,
    +                guest_memory: &guest_memory,
    +                cpuid: &[],
    +                vtl0_alias_map: None,
    +                fault_resolver: None,
    +            })
    +            .unwrap();
    +        assert!(
    +            !partition.caps().hv1,
    +            "a microVM partition has no Hyper-V enlightenments"
    +        );
    +        f(&partition, binders)
    +    })
    +}
    +
    +/// A device's MSI to `apic_id` (physical destination, edge).
    +fn msi(mode: DeliveryMode, apic_id: u32, vector: u8) -> MsiRequest {
    +    MsiRequest::new_x86(mode, apic_id, false, vector, false)
    +}
    +
    +/// Sets `vp`'s mp_state to `mp_state` (KVM_SET_MP_STATE), the rest of its
    +/// activity at reset.
    +fn set_mp_state(vp: &mut KvmProcessor<'_>, mp_state: vp::MpState) {
    +    vp.access_state(Vtl::Vtl0)
    +        .set_activity(&vp::Activity {
    +            mp_state,
    +            ..Default::default()
    +        })
    +        .unwrap();
    +}
    +
    +/// The mp_state that `vp`'s activity reads (KVM_GET_MP_STATE).
    +fn mp_state(vp: &mut KvmProcessor<'_>) -> vp::MpState {
    +    vp.access_state(Vtl::Vtl0).activity().unwrap().mp_state
    +}
    +
    +/// BUG-21 (R12) witness, BUG-09's class for a PV kick: an MSI or IO-APIC
    +/// route with delivery mode REMOTE_READ (3, reserved on today's x86; KVM uses
    +/// it for its PV kick, x86.c 9841-9850) reaches a stopped VP at once (it is
    +/// not deferred) and sets KVM's `pv_unhalted` (lapic.c 1304-1309), so KVM
    +/// reports a halted VP as runnable (x86.c 11775-11778): a save after the VP's
    +/// Stop reads mp_state Running where its Stop recorded Halted. The guest
    +/// can program this mode in an IO-APIC entry; this test uses the same MSI path.
    +#[test]
    +#[ignore = "requires /dev/kvm"]
    +fn remote_read_after_stop_keeps_a_halted_vps_mp_state() {
    +    with_partition(1, |partition, mut binders| {
    +        let mut vp = binders[0].bind().unwrap();
    +        let apic_id = vp.inner.vp_info.apic_id;
    +        set_mp_state(&mut vp, vp::MpState::Halted);
    +        assert_eq!(mp_state(&mut vp), vp::MpState::Halted, "at the Stop");
    +
    +        partition.request_msi(Vtl::Vtl0, msi(DeliveryMode::REMOTE_READ, apic_id, 0));
    +        assert_eq!(
    +            mp_state(&mut vp),
    +            vp::MpState::Halted,
    +            "a device's REMOTE_READ changed a stopped VP's mp_state"
    +        );
    +    })
    +}
    +
    +/// BUG-21's fix: KvmMsi::new drops delivery mode REMOTE_READ (011, reserved)
    +/// and keeps every other mode as before: the same delivery mode, vector,
    +/// trigger mode and destination, for each of the other seven modes, edge and
    +/// level.
    +#[test]
    +fn kvm_msi_new_drops_only_remote_read() {
    +    for mode in 0..8u8 {
    +        for level in [false, true] {
    +            let request = MsiRequest::new_x86(DeliveryMode(mode), 5, false, 0x40, level);
    +            let msi = KvmMsi::new(request);
    +            if mode == DeliveryMode::REMOTE_READ.0 {
    +                assert!(msi.is_none(), "REMOTE_READ is dropped");
    +                continue;
    +            }
    +            let msi = msi.unwrap_or_else(|| panic!("mode {mode} is kept"));
    +            let data = MsiData::from(msi.data);
    +            let address = MsiAddress::from(msi.address_lo);
    +            assert_eq!(
    +                (
    +                    data.delivery_mode(),
    +                    data.vector(),
    +                    data.trigger_mode_level(),
    +                    address.destination()
    +                ),
    +                (mode, 0x40, level, 5),
    +                "mode {mode}, level {level}"
    +            );
    +        }
    +    }
    +}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Labels

No labels
No labels

Type

Projects

No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions