From bf30d4a8fabefdf6402eacf7f4df7a64a6fd29d0 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 6 Nov 2026 18:24:32 -0800 Subject: [PATCH 1/2] vpci: revoke a bus's channel without borrowing the bus VpciBus::revoke was an async fn that borrowed the bus mutably until the revocation of its VMBus channel completed. That revocation takes a round trip through the VMBus server, which drops the channel, and then, if the guest opened the channel, the channel's close, which powers the device off, unregisters its interrupts, and clears its BARs, awaiting any config-space write that the device defers. A caller therefore had to keep exclusive access to the bus across that wait. DynamicVpciDevice::remove, the only caller, does so by closing the bus's CloseableMutex, although the bus stays registered for VP accesses, and its state unit can still lock it, until the bus is removed. Make revoke synchronous: it moves the channel out, records the bus as revoked, and returns a Send future that waits for the revocation without borrowing the bus. A caller can take the future under a short lock and await it after releasing the lock. Callers that await the result directly still compile, and an offer in progress is handled as before. Tests: a new test offers a bus's channel to a VMBus that holds it, so that the channel's revocation stays pending. The bus records the revocation at once; while the revocation waits, the bus locks normally and serves a config-space read through its MMIO intercept. The revocation completes once the VMBus releases the channel, and revoking again completes at once. See microsoft/nvx#1156. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vm/devices/pci/vpci/src/bus.rs | 99 +++++++++++++++++++++++++++++++--- 1 file changed, 93 insertions(+), 6 deletions(-) diff --git a/vm/devices/pci/vpci/src/bus.rs b/vm/devices/pci/vpci/src/bus.rs index 460e0e60ae5..c7810fcbaaa 100644 --- a/vm/devices/pci/vpci/src/bus.rs +++ b/vm/devices/pci/vpci/src/bus.rs @@ -236,15 +236,28 @@ impl VpciBus { self.eject.clone() } - /// Revokes the VMBus channel and waits for revocation to complete. + /// Revokes the VMBus channel, returning a future that completes when the + /// revocation does. + /// + /// The bus records the revocation before this returns, and the returned + /// future does not borrow it. Release any lock on the bus before awaiting + /// the future: the revocation waits for the VMBus server and for the + /// channel to close, and VPs and the bus's state unit can access the bus + /// until it is removed. /// /// If an offer is in progress, the offering task revokes its channel when /// the offer completes. - pub async fn revoke(&mut self) { - if let VpciBusChannelState::Offered { channel } = - std::mem::replace(&mut self.channel, VpciBusChannelState::Revoked) - { - channel.revoke().await; + pub fn revoke(&mut self) -> impl Future + Send + use<> { + let channel = match std::mem::replace(&mut self.channel, VpciBusChannelState::Revoked) { + VpciBusChannelState::Offered { channel } => Some(channel), + VpciBusChannelState::Unoffered + | VpciBusChannelState::Offering + | VpciBusChannelState::Revoked => None, + }; + async move { + if let Some(channel) = channel { + channel.revoke().await; + } } } } @@ -538,6 +551,7 @@ mod tests { use chipset_device::pci::PciConfigSpace; use chipset_device::poll_device::PollDevice; use closeable_mutex::CloseableMutex; + use guestmem::GuestMemory; use guid::Guid; use inspect::InspectMut; use pal_async::DefaultDriver; @@ -549,6 +563,10 @@ mod tests { use std::task::Context; use std::task::Poll; use test_with_tracing::test; + use vmbus_channel::bus::OfferInput; + use vmbus_channel::bus::OfferResources; + use vmbus_channel::bus::ParentBus; + use vmcore::vm_task::SingleDriverBackend; use vmcore::vpci_msi::VpciInterruptMapper; const BASE_ADDR: u64 = 0x1000_0000; @@ -994,4 +1012,73 @@ mod tests { assert!(rig.device.lock().pending_write.is_none()); assert_eq!(rig.device.lock().writes.len(), writes_before); } + + /// A VMBus that holds each offered channel until the test releases it, so + /// that the channel's revocation stays pending until then. + #[derive(Clone, Default)] + struct HeldChannels(Arc>>); + + #[async_trait::async_trait] + impl ParentBus for HeldChannels { + async fn add_child(&self, request: OfferInput) -> anyhow::Result { + self.0.lock().push(request); + Ok(OfferResources::new(GuestMemory::empty(), None)) + } + + fn clone_bus(&self) -> Box { + Box::new(self.clone()) + } + + fn use_event(&self) -> bool { + false + } + } + + #[async_test] + async fn bus_stays_accessible_while_its_channel_is_revoked(driver: DefaultDriver) { + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let vmbus = HeldChannels::default(); + let device = Arc::new(CloseableMutex::new(DeferWriteDevice::new())); + let bus = VpciBus::new( + &driver_source, + VpciBusConfig { + instance_id: Guid::new_random(), + vtom: None, + vnode: None, + }, + device.clone(), + &mut ExternallyManagedMmioIntercepts, + &vmbus, + VpciInterruptMapper::new(TestVpciInterruptController::new()), + ) + .await + .unwrap(); + bus.bus_device.config_space_offset().set(BASE_ADDR); + let bus = Arc::new(CloseableMutex::new(bus)); + let mut offer = vmbus.0.lock().pop().expect("the bus offered its channel"); + + // The revocation does not borrow the bus, so it can run as a task of + // its own. The bus records it at once. + let revoke = driver.spawn("revoke", bus.lock().revoke()); + assert!(matches!(bus.lock().channel, VpciBusChannelState::Revoked)); + // The VMBus is asked to rescind the channel... + assert!(offer.server_request_recv.recv().await.is_err()); + // ...but until it releases the channel, the bus still serves VPs. + device.lock().read_value = 0x1234_5678; + let mut data = [0; 4]; + assert!(matches!( + bus.lock() + .supports_mmio() + .unwrap() + .mmio_read(VpciTestRig::config_addr(0), &mut data), + IoResult::Ok + )); + assert_eq!(u32::from_ne_bytes(data), 0x1234_5678); + assert!(!revoke.is_finished()); + + drop(offer); + revoke.await; + // A revoked bus has nothing left to revoke. + bus.lock().revoke().await; + } } From 2158cbffeb6f0d9ec90748767da5b7cacf7f471f Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 6 Nov 2026 18:24:40 -0800 Subject: [PATCH 2/2] vmm_core: keep a removed VPCI bus open while its channel is revoked DynamicVpciDevice::remove closed the VPCI bus's CloseableMutex and kept it closed while awaiting VpciBus::revoke, which waits for the VMBus server to drop the channel and for the channel to close. The bus stays registered as a state unit and as an MMIO device until vpci_unit.remove, so in that window: - Inspecting the VM, which OpenVMM's worker serves from a separate task even while a removal is pending, reached the bus's state unit, whose inspect_mut locks the bus with lock(), and panicked with "lock should not be closed", which aborts the process. The unit's poll loop locks the bus the same way when a deferred config-space access completes. - VP accesses to the bus failed with IoError::NoResponse, because chipset dispatch uses lock_if_open as of #4863. Before #4863, they panicked too. Take the revocation future under a short ordinary lock, which VpciBus::revoke now allows, and await it after releasing the lock. The bus is still revoked, then removed, before its PCI device. Tests: a new test builds a dynamic VPCI device on a chipset and offers its channel to a VMBus that holds it, so that the removal waits for the revocation. Meanwhile, inspecting the bus's state unit reports the revoked channel, and the bus locks as chipset dispatch requires. Releasing the channel completes the removal, which closes both devices permanently. Without this change, the inspection panics with "lock should not be closed". See microsoft/nvx#1156. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Cargo.lock | 2 + vmm_core/Cargo.toml | 3 + vmm_core/src/device_builder.rs | 232 ++++++++++++++++++++++++++++++++- 3 files changed, 236 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index b2c2dfb39b9..23a5776be6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11492,6 +11492,7 @@ dependencies = [ "build_rs_guest_arch", "cache_topology", "chipset", + "chipset_device", "chipset_device_resources", "chipset_resources", "closeable_mutex", @@ -11499,6 +11500,7 @@ dependencies = [ "futures", "futures-concurrency", "guestmem", + "guid", "hcl_compat_uefi_nvram_storage", "hvdef", "hyperv_dump", diff --git a/vmm_core/Cargo.toml b/vmm_core/Cargo.toml index de8f3943f3e..c86d2c594c7 100644 --- a/vmm_core/Cargo.toml +++ b/vmm_core/Cargo.toml @@ -63,6 +63,9 @@ cvm_tracing.workspace = true iced-x86 = { optional = true, workspace = true, features = ["std", "fast_fmt"] } [dev-dependencies] +chipset_device.workspace = true +guid.workspace = true +inspect = { workspace = true, features = ["initiate"] } test_with_tracing.workspace = true [build-dependencies] diff --git a/vmm_core/src/device_builder.rs b/vmm_core/src/device_builder.rs index 089ac1444c9..99f5a71ccc8 100644 --- a/vmm_core/src/device_builder.rs +++ b/vmm_core/src/device_builder.rs @@ -55,8 +55,13 @@ impl DynamicVpciDevice { } /// Removes the VPCI bus before removing its underlying PCI device. + /// + /// The bus remains available to VPs and to inspection while its VMBus + /// channel is revoked. pub async fn remove(self) { - self.vpci_bus.close().revoke().await; + // Release the lock before waiting for the revocation. + let revoke = self.vpci_bus.lock().revoke(); + revoke.await; self.vpci_unit.remove().await; self.pci_unit.remove().await; } @@ -274,3 +279,228 @@ pub async fn resolve_and_add_pci_device( Ok(device) } + +#[cfg(test)] +mod tests { + use super::DynamicVpciDevice; + use chipset_device::ChipsetDevice; + use chipset_device::io::IoResult; + use chipset_device::pci::ByteEnabledDwordRead; + use chipset_device::pci::ByteEnabledDwordWrite; + use chipset_device::pci::PciConfigSpace; + use futures::poll; + use guestmem::GuestMemory; + use inspect::InspectMut; + use pal_async::DefaultDriver; + use pal_async::async_test; + use parking_lot::Mutex; + use state_unit::StateUnits; + use std::pin::pin; + use std::sync::Arc; + use test_with_tracing::test; + use vm_resource::ResourceResolver; + use vmbus_channel::bus::OfferInput; + use vmbus_channel::bus::OfferResources; + use vmbus_channel::bus::ParentBus; + use vmcore::device_state::ChangeDeviceState; + use vmcore::save_restore::NoSavedState; + use vmcore::save_restore::RestoreError; + use vmcore::save_restore::SaveError; + use vmcore::save_restore::SaveRestore; + use vmcore::vm_task::SingleDriverBackend; + use vmcore::vm_task::VmTaskDriverSource; + use vmcore::vmtime::VmTime; + use vmcore::vmtime::VmTimeKeeper; + use vmcore::vpci_msi::MapVpciInterrupt; + use vmcore::vpci_msi::MsiAddressData; + use vmcore::vpci_msi::RegisterInterruptError; + use vmcore::vpci_msi::VpciInterruptMapper; + use vmcore::vpci_msi::VpciInterruptParameters; + use vmotherboard::BaseChipsetBuilder; + use vmotherboard::DebugEventHandler; + use vmotherboard::PowerEvent; + use vmotherboard::PowerEventHandler; + use vmotherboard::options::BaseChipsetDevices; + use vmotherboard::options::BaseChipsetFoundation; + use vpci::bus::VpciBus; + use vpci::bus::VpciBusConfig; + + /// A PCI device without BARs. + #[derive(InspectMut)] + #[inspect(skip)] + struct TestPciDevice; + + impl ChipsetDevice for TestPciDevice { + fn supports_pci(&mut self) -> Option<&mut dyn PciConfigSpace> { + Some(self) + } + } + + impl PciConfigSpace for TestPciDevice { + fn pci_cfg_read(&mut self, _offset: u16, mut value: ByteEnabledDwordRead<'_>) -> IoResult { + value.set(0); + IoResult::Ok + } + + fn pci_cfg_write(&mut self, _offset: u16, _value: ByteEnabledDwordWrite) -> IoResult { + IoResult::Ok + } + } + + impl ChangeDeviceState for TestPciDevice { + fn start(&mut self) {} + + async fn stop(&mut self) {} + + async fn reset(&mut self) {} + } + + impl SaveRestore for TestPciDevice { + type SavedState = NoSavedState; + + fn save(&mut self) -> Result { + Ok(NoSavedState) + } + + fn restore(&mut self, NoSavedState: NoSavedState) -> Result<(), RestoreError> { + Ok(()) + } + } + + struct NoInterrupts; + + impl MapVpciInterrupt for NoInterrupts { + async fn register_interrupt( + &self, + _vector_count: u32, + _params: &VpciInterruptParameters<'_>, + ) -> Result { + Err(RegisterInterruptError::new("interrupts are not supported")) + } + + async fn unregister_interrupt(&self, _address: u64, _data: u32) {} + } + + struct NoEvents; + + impl PowerEventHandler for NoEvents { + fn on_power_event(&self, _evt: PowerEvent) {} + } + + impl DebugEventHandler for NoEvents { + fn on_debug_break(&self, _vp: Option) {} + } + + /// A VMBus that holds each offered channel until the test releases it, so + /// that the channel's revocation stays pending until then. + #[derive(Clone, Default)] + struct HeldChannels(Arc>>); + + impl HeldChannels { + fn release(&self) { + self.0.lock().clear(); + } + } + + #[async_trait::async_trait] + impl ParentBus for HeldChannels { + async fn add_child(&self, request: OfferInput) -> anyhow::Result { + self.0.lock().push(request); + Ok(OfferResources::new(GuestMemory::empty(), None)) + } + + fn clone_bus(&self) -> Box { + Box::new(self.clone()) + } + + fn use_event(&self) -> bool { + false + } + } + + #[async_test] + async fn removal_keeps_the_bus_open_while_its_channel_is_revoked(driver: DefaultDriver) { + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let units = StateUnits::new(); + let (vmtime_send, _vmtime_recv) = mesh::channel(); + let vmtime_unit = units.add("vmtime").build(vmtime_send).unwrap(); + let vmtime_keeper = VmTimeKeeper::new(&driver, VmTime::from_100ns(0)); + let vmtime = vmtime_keeper.builder().build(&driver).await.unwrap(); + let events = Arc::new(NoEvents); + let chipset = BaseChipsetBuilder::new( + BaseChipsetFoundation { + is_restoring: false, + untrusted_dma_memory: GuestMemory::empty(), + trusted_vtl0_dma_memory: GuestMemory::empty(), + power_event_handler: events.clone(), + debug_event_handler: events, + vmtime: &vmtime, + vmtime_unit: &vmtime_unit, + doorbell_registration: None, + }, + BaseChipsetDevices::empty(), + ) + .build(&driver_source, &units, &ResourceResolver::new()) + .await + .unwrap(); + let (_chipset, devices) = chipset.chipset_builder.build().unwrap(); + + // Assemble the device as `build_dynamic_vpci_device` does, but offer + // its channel to a VMBus that the test controls. + let (pci_unit, pci_device) = devices + .add_dyn_device(&driver_source, &units, "pci", async |_| { + anyhow::Ok(TestPciDevice) + }) + .await + .unwrap(); + let mut pending_offer = None; + let (vpci_unit, vpci_bus) = devices + .add_dyn_device(&driver_source, &units, "vpci", async |register_mmio| { + let (bus, offer) = VpciBus::new_unoffered( + VpciBusConfig { + instance_id: guid::guid!("6f1bc3e5-07a5-4b4f-9d36-0c5e4ad0f6f1"), + vtom: None, + vnode: None, + }, + pci_device.clone(), + register_mmio, + VpciInterruptMapper::new(Arc::new(NoInterrupts)), + )?; + pending_offer = Some(offer); + anyhow::Ok(bus) + }) + .await + .unwrap(); + let vmbus = HeldChannels::default(); + pending_offer + .unwrap() + .offer_registered(&vpci_bus, &driver_source, &vmbus, false) + .await + .unwrap(); + let eject = vpci_bus.lock().eject_control(); + let device = DynamicVpciDevice { + pci_unit, + vpci_unit, + eject, + vpci_bus: vpci_bus.clone(), + }; + + // Removal revokes the channel and then waits until the VMBus releases + // it. + let mut removal = pin!(device.remove()); + assert!(poll!(&mut removal).is_pending()); + + // Meanwhile, the bus still serves inspection through its state unit... + let mut inspection = inspect::inspect("vpci/state", &units); + inspection.resolve().await; + assert_eq!(inspection.results().json().to_string(), r#""revoked""#); + // ...and VP accesses, which chipset dispatch makes only to an open bus. + assert!(vpci_bus.lock_if_open().is_some()); + assert!(poll!(&mut removal).is_pending()); + + vmbus.release(); + removal.await; + assert!(vpci_bus.lock_if_open().is_none()); + assert!(pci_device.lock_if_open().is_none()); + } +}