Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -11492,13 +11492,15 @@ dependencies = [
"build_rs_guest_arch",
"cache_topology",
"chipset",
"chipset_device",
"chipset_device_resources",
"chipset_resources",
"closeable_mutex",
"cvm_tracing",
"futures",
"futures-concurrency",
"guestmem",
"guid",
"hcl_compat_uefi_nvram_storage",
"hvdef",
"hyperv_dump",
Expand Down
99 changes: 93 additions & 6 deletions vm/devices/pci/vpci/src/bus.rs
Original file line number Diff line number Diff line change
Expand Up @@ -236,15 +236,28 @@ impl VpciBus {
self.eject.clone()
}

/// Revokes the VMBus channel and waits for revocation to complete.
/// Revokes the VMBus channel, returning a future that completes when the
/// revocation does.
///
/// The bus records the revocation before this returns, and the returned
/// future does not borrow it. Release any lock on the bus before awaiting
/// the future: the revocation waits for the VMBus server and for the
/// channel to close, and VPs and the bus's state unit can access the bus
/// until it is removed.
///
/// If an offer is in progress, the offering task revokes its channel when
/// the offer completes.
pub async fn revoke(&mut self) {
if let VpciBusChannelState::Offered { channel } =
std::mem::replace(&mut self.channel, VpciBusChannelState::Revoked)
{
channel.revoke().await;
pub fn revoke(&mut self) -> impl Future<Output = ()> + Send + use<> {
let channel = match std::mem::replace(&mut self.channel, VpciBusChannelState::Revoked) {
VpciBusChannelState::Offered { channel } => Some(channel),
VpciBusChannelState::Unoffered
| VpciBusChannelState::Offering
| VpciBusChannelState::Revoked => None,
};
async move {
if let Some(channel) = channel {
channel.revoke().await;
}
}
}
}
Expand Down Expand Up @@ -538,6 +551,7 @@ mod tests {
use chipset_device::pci::PciConfigSpace;
use chipset_device::poll_device::PollDevice;
use closeable_mutex::CloseableMutex;
use guestmem::GuestMemory;
use guid::Guid;
use inspect::InspectMut;
use pal_async::DefaultDriver;
Expand All @@ -549,6 +563,10 @@ mod tests {
use std::task::Context;
use std::task::Poll;
use test_with_tracing::test;
use vmbus_channel::bus::OfferInput;
use vmbus_channel::bus::OfferResources;
use vmbus_channel::bus::ParentBus;
use vmcore::vm_task::SingleDriverBackend;
use vmcore::vpci_msi::VpciInterruptMapper;

const BASE_ADDR: u64 = 0x1000_0000;
Expand Down Expand Up @@ -994,4 +1012,73 @@ mod tests {
assert!(rig.device.lock().pending_write.is_none());
assert_eq!(rig.device.lock().writes.len(), writes_before);
}

/// A VMBus that holds each offered channel until the test releases it, so
/// that the channel's revocation stays pending until then.
#[derive(Clone, Default)]
struct HeldChannels(Arc<Mutex<Vec<OfferInput>>>);

#[async_trait::async_trait]
impl ParentBus for HeldChannels {
async fn add_child(&self, request: OfferInput) -> anyhow::Result<OfferResources> {
self.0.lock().push(request);
Ok(OfferResources::new(GuestMemory::empty(), None))
}

fn clone_bus(&self) -> Box<dyn ParentBus> {
Box::new(self.clone())
}

fn use_event(&self) -> bool {
false
}
}

#[async_test]
async fn bus_stays_accessible_while_its_channel_is_revoked(driver: DefaultDriver) {
let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone()));
let vmbus = HeldChannels::default();
let device = Arc::new(CloseableMutex::new(DeferWriteDevice::new()));
let bus = VpciBus::new(
&driver_source,
VpciBusConfig {
instance_id: Guid::new_random(),
vtom: None,
vnode: None,
},
device.clone(),
&mut ExternallyManagedMmioIntercepts,
&vmbus,
VpciInterruptMapper::new(TestVpciInterruptController::new()),
)
.await
.unwrap();
bus.bus_device.config_space_offset().set(BASE_ADDR);
let bus = Arc::new(CloseableMutex::new(bus));
let mut offer = vmbus.0.lock().pop().expect("the bus offered its channel");

// The revocation does not borrow the bus, so it can run as a task of
// its own. The bus records it at once.
let revoke = driver.spawn("revoke", bus.lock().revoke());
assert!(matches!(bus.lock().channel, VpciBusChannelState::Revoked));
// The VMBus is asked to rescind the channel...
assert!(offer.server_request_recv.recv().await.is_err());
// ...but until it releases the channel, the bus still serves VPs.
device.lock().read_value = 0x1234_5678;
let mut data = [0; 4];
assert!(matches!(
bus.lock()
.supports_mmio()
.unwrap()
.mmio_read(VpciTestRig::config_addr(0), &mut data),
IoResult::Ok
));
assert_eq!(u32::from_ne_bytes(data), 0x1234_5678);
assert!(!revoke.is_finished());

drop(offer);
revoke.await;
// A revoked bus has nothing left to revoke.
bus.lock().revoke().await;
}
}
3 changes: 3 additions & 0 deletions vmm_core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ cvm_tracing.workspace = true
iced-x86 = { optional = true, workspace = true, features = ["std", "fast_fmt"] }

[dev-dependencies]
chipset_device.workspace = true
guid.workspace = true
inspect = { workspace = true, features = ["initiate"] }
test_with_tracing.workspace = true

[build-dependencies]
Expand Down
232 changes: 231 additions & 1 deletion vmm_core/src/device_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,13 @@ impl DynamicVpciDevice {
}

/// Removes the VPCI bus before removing its underlying PCI device.
///
/// The bus remains available to VPs and to inspection while its VMBus
/// channel is revoked.
pub async fn remove(self) {
self.vpci_bus.close().revoke().await;
// Release the lock before waiting for the revocation.
let revoke = self.vpci_bus.lock().revoke();
revoke.await;
self.vpci_unit.remove().await;
self.pci_unit.remove().await;
}
Expand Down Expand Up @@ -274,3 +279,228 @@ pub async fn resolve_and_add_pci_device(

Ok(device)
}

#[cfg(test)]
mod tests {
use super::DynamicVpciDevice;
use chipset_device::ChipsetDevice;
use chipset_device::io::IoResult;
use chipset_device::pci::ByteEnabledDwordRead;
use chipset_device::pci::ByteEnabledDwordWrite;
use chipset_device::pci::PciConfigSpace;
use futures::poll;
use guestmem::GuestMemory;
use inspect::InspectMut;
use pal_async::DefaultDriver;
use pal_async::async_test;
use parking_lot::Mutex;
use state_unit::StateUnits;
use std::pin::pin;
use std::sync::Arc;
use test_with_tracing::test;
use vm_resource::ResourceResolver;
use vmbus_channel::bus::OfferInput;
use vmbus_channel::bus::OfferResources;
use vmbus_channel::bus::ParentBus;
use vmcore::device_state::ChangeDeviceState;
use vmcore::save_restore::NoSavedState;
use vmcore::save_restore::RestoreError;
use vmcore::save_restore::SaveError;
use vmcore::save_restore::SaveRestore;
use vmcore::vm_task::SingleDriverBackend;
use vmcore::vm_task::VmTaskDriverSource;
use vmcore::vmtime::VmTime;
use vmcore::vmtime::VmTimeKeeper;
use vmcore::vpci_msi::MapVpciInterrupt;
use vmcore::vpci_msi::MsiAddressData;
use vmcore::vpci_msi::RegisterInterruptError;
use vmcore::vpci_msi::VpciInterruptMapper;
use vmcore::vpci_msi::VpciInterruptParameters;
use vmotherboard::BaseChipsetBuilder;
use vmotherboard::DebugEventHandler;
use vmotherboard::PowerEvent;
use vmotherboard::PowerEventHandler;
use vmotherboard::options::BaseChipsetDevices;
use vmotherboard::options::BaseChipsetFoundation;
use vpci::bus::VpciBus;
use vpci::bus::VpciBusConfig;

/// A PCI device without BARs.
#[derive(InspectMut)]
#[inspect(skip)]
struct TestPciDevice;

impl ChipsetDevice for TestPciDevice {
fn supports_pci(&mut self) -> Option<&mut dyn PciConfigSpace> {
Some(self)
}
}

impl PciConfigSpace for TestPciDevice {
fn pci_cfg_read(&mut self, _offset: u16, mut value: ByteEnabledDwordRead<'_>) -> IoResult {
value.set(0);
IoResult::Ok
}

fn pci_cfg_write(&mut self, _offset: u16, _value: ByteEnabledDwordWrite) -> IoResult {
IoResult::Ok
}
}

impl ChangeDeviceState for TestPciDevice {
fn start(&mut self) {}

async fn stop(&mut self) {}

async fn reset(&mut self) {}
}

impl SaveRestore for TestPciDevice {
type SavedState = NoSavedState;

fn save(&mut self) -> Result<NoSavedState, SaveError> {
Ok(NoSavedState)
}

fn restore(&mut self, NoSavedState: NoSavedState) -> Result<(), RestoreError> {
Ok(())
}
}

struct NoInterrupts;

impl MapVpciInterrupt for NoInterrupts {
async fn register_interrupt(
&self,
_vector_count: u32,
_params: &VpciInterruptParameters<'_>,
) -> Result<MsiAddressData, RegisterInterruptError> {
Err(RegisterInterruptError::new("interrupts are not supported"))
}

async fn unregister_interrupt(&self, _address: u64, _data: u32) {}
}

struct NoEvents;

impl PowerEventHandler for NoEvents {
fn on_power_event(&self, _evt: PowerEvent) {}
}

impl DebugEventHandler for NoEvents {
fn on_debug_break(&self, _vp: Option<u32>) {}
}

/// A VMBus that holds each offered channel until the test releases it, so
/// that the channel's revocation stays pending until then.
#[derive(Clone, Default)]
struct HeldChannels(Arc<Mutex<Vec<OfferInput>>>);

impl HeldChannels {
fn release(&self) {
self.0.lock().clear();
}
}

#[async_trait::async_trait]
impl ParentBus for HeldChannels {
async fn add_child(&self, request: OfferInput) -> anyhow::Result<OfferResources> {
self.0.lock().push(request);
Ok(OfferResources::new(GuestMemory::empty(), None))
}

fn clone_bus(&self) -> Box<dyn ParentBus> {
Box::new(self.clone())
}

fn use_event(&self) -> bool {
false
}
}

#[async_test]
async fn removal_keeps_the_bus_open_while_its_channel_is_revoked(driver: DefaultDriver) {
let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone()));
let units = StateUnits::new();
let (vmtime_send, _vmtime_recv) = mesh::channel();
let vmtime_unit = units.add("vmtime").build(vmtime_send).unwrap();
let vmtime_keeper = VmTimeKeeper::new(&driver, VmTime::from_100ns(0));
let vmtime = vmtime_keeper.builder().build(&driver).await.unwrap();
let events = Arc::new(NoEvents);
let chipset = BaseChipsetBuilder::new(
BaseChipsetFoundation {
is_restoring: false,
untrusted_dma_memory: GuestMemory::empty(),
trusted_vtl0_dma_memory: GuestMemory::empty(),
power_event_handler: events.clone(),
debug_event_handler: events,
vmtime: &vmtime,
vmtime_unit: &vmtime_unit,
doorbell_registration: None,
},
BaseChipsetDevices::empty(),
)
.build(&driver_source, &units, &ResourceResolver::new())
.await
.unwrap();
let (_chipset, devices) = chipset.chipset_builder.build().unwrap();

// Assemble the device as `build_dynamic_vpci_device` does, but offer
// its channel to a VMBus that the test controls.
let (pci_unit, pci_device) = devices
.add_dyn_device(&driver_source, &units, "pci", async |_| {
anyhow::Ok(TestPciDevice)
})
.await
.unwrap();
let mut pending_offer = None;
let (vpci_unit, vpci_bus) = devices
.add_dyn_device(&driver_source, &units, "vpci", async |register_mmio| {
let (bus, offer) = VpciBus::new_unoffered(
VpciBusConfig {
instance_id: guid::guid!("6f1bc3e5-07a5-4b4f-9d36-0c5e4ad0f6f1"),
vtom: None,
vnode: None,
},
pci_device.clone(),
register_mmio,
VpciInterruptMapper::new(Arc::new(NoInterrupts)),
)?;
pending_offer = Some(offer);
anyhow::Ok(bus)
})
.await
.unwrap();
let vmbus = HeldChannels::default();
pending_offer
.unwrap()
.offer_registered(&vpci_bus, &driver_source, &vmbus, false)
.await
.unwrap();
let eject = vpci_bus.lock().eject_control();
let device = DynamicVpciDevice {
pci_unit,
vpci_unit,
eject,
vpci_bus: vpci_bus.clone(),
};

// Removal revokes the channel and then waits until the VMBus releases
// it.
let mut removal = pin!(device.remove());
assert!(poll!(&mut removal).is_pending());

// Meanwhile, the bus still serves inspection through its state unit...
let mut inspection = inspect::inspect("vpci/state", &units);
inspection.resolve().await;
assert_eq!(inspection.results().json().to_string(), r#""revoked""#);
// ...and VP accesses, which chipset dispatch makes only to an open bus.
assert!(vpci_bus.lock_if_open().is_some());
assert!(poll!(&mut removal).is_pending());

vmbus.release();
removal.await;
assert!(vpci_bus.lock_if_open().is_none());
assert!(pci_device.lock_if_open().is_none());
}
}
Loading