diff --git a/Cargo.lock b/Cargo.lock index 9dbcf475419..1b70b448053 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -653,6 +653,7 @@ dependencies = [ name = "chipset_device_resources" version = "0.0.0" dependencies = [ + "anyhow", "async-trait", "chipset_device", "guestmem", diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index 20eebc55c06..a336eac9e08 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -79,7 +79,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc; #[derive(MeshPayload)] pub enum UhVmRpc { Pause(Rpc<(), bool>), - Resume(Rpc<(), bool>), + Resume(FailableRpc<(), bool>), Save(FailableRpc<(), Vec>), ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume PacketCapture(FailableRpc, PacketCaptureParams>), @@ -233,9 +233,12 @@ impl LoadedVm { correlation_id: Option, mut vm_rpc: mesh::Receiver, mut worker_rpc: mesh::Receiver>, - ) -> Option> { + ) -> anyhow::Result>> { if autostart_vps { - self.start(correlation_id).await; + if let Err(error) = self.start(correlation_id).await { + self.teardown().await; + return Err(error.context("failed to autostart VM")); + } } // VTL2 settings services @@ -323,7 +326,17 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - let state = async { + enum RestartOutcome { + Restart(R, S), + // The restart failed, but the VM is back in its + // previous state, so this worker can keep serving. + Recovered, + // The restart failed and the VM could not be + // resumed, so this worker cannot serve any more. + Unrecoverable, + } + + let outcome = async { let running = self.stop().await; match self .save( @@ -334,26 +347,40 @@ impl LoadedVm { ) .await { - Ok(servicing_state) => Some((rpc, servicing_state)), + Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state), Err(err) => { if running { - self.start(None).await; + if let Err(start_error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + start_error.context(format!( + "worker restart failed ({err:#}) and the VM could not resume" + )), + ))); + return RestartOutcome::Unrecoverable; + } } rpc.complete(Err(RemoteError::new(err))); - None + RestartOutcome::Recovered } } } .instrument(tracing::info_span!("restart", CVM_ALLOWED)) .await; - if let Some((rpc, servicing_state)) = state { - break Some(LoadedVmState { - restart_rpc: rpc, - servicing_state, - vm_rpc, - control_send: self.control_send.lock().take().unwrap(), - }); + match outcome { + RestartOutcome::Restart(rpc, servicing_state) => { + break Some(LoadedVmState { + restart_rpc: rpc, + servicing_state, + vm_rpc, + control_send: self.control_send.lock().take().unwrap(), + }); + } + RestartOutcome::Recovered => {} + // The error has already completed the RPC. Tear + // down through the common path rather than keep a + // worker whose state units can no longer run. + RestartOutcome::Unrecoverable => break None, } } WorkerRpc::Inspect(deferred) => deferred.respond(|resp| { @@ -403,34 +430,36 @@ impl LoadedVm { } Event::UhVmRpc(msg) => match msg { UhVmRpc::Resume(rpc) => { - rpc.handle(async |()| { - if !self.state_units.is_running() { - self.start(None).await; - true - } else { - false - } - }) - .await + if self.state_units.is_running() { + rpc.complete(Ok(false)); + } else if let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM"), + ))); + } else { + rpc.complete(Ok(true)); + } } UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await, UhVmRpc::Save(rpc) => { - rpc.handle_failable(async |()| { - let running = self.stop().await; - let r = self - .save( - SaveReason::Diagnostic, - None, - KeepAliveConfig::Disabled, - KeepAliveConfig::Disabled, - ) - .await; - if running { - self.start(None).await; - } - r.map(mesh::payload::encode) - }) - .await + let running = self.stop().await; + let result = self + .save( + SaveReason::Diagnostic, + None, + KeepAliveConfig::Disabled, + KeepAliveConfig::Disabled, + ) + .await; + if running && let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM after save"), + ))); + // The state units rolled back and cannot run + // again, so tear down rather than keep this worker. + break None; + } + rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new)); } UhVmRpc::ClearHalt(rpc) => { rpc.handle(async |()| self.partition_unit.clear_halt().await) @@ -582,6 +611,12 @@ impl LoadedVm { } }; + self.teardown().await; + + Ok(state) + } + + async fn teardown(mut self) { // Drain resealing before tearing down any storage transport. Removing // the unit closes its request stream; in-flight I/O finishes first. if let Some(resealer) = self.hardware_reseal.take() { @@ -599,8 +634,6 @@ impl LoadedVm { if let Some(vmbus) = self.vmbus_server { vmbus.remove().await.shutdown().await; } - - state } /// Handles a servicing request from the host. @@ -651,7 +684,9 @@ impl LoadedVm { .context("failed to notify host of servicing-while-paused failure")?; if running { - self.start(Some(correlation_id)).await; + self.start(Some(correlation_id)) + .await + .context("failed to resume VM after servicing failure")?; } false } @@ -881,8 +916,8 @@ impl LoadedVm { } } - async fn start(&mut self, correlation_id: Option) { - self.state_units.start().await; + async fn start(&mut self, correlation_id: Option) -> anyhow::Result<()> { + self.state_units.start().await?; // Log the boot/blackout time. let reference_time = ReferenceTime::new(self.partition.reference_time()); @@ -908,6 +943,8 @@ impl LoadedVm { "starting VM" ) } + + Ok(()) } /// Returns true if the VM was previously running. @@ -1134,7 +1171,20 @@ impl LoadedVm { ) .await?; - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + if let Err(remove_error) = self + .network_settings + .as_mut() + .unwrap() + .remove_network(instance_id) + .await + { + return Err(error.context(format!( + "failed to remove network after start failure: {remove_error:#}" + ))); + } + return Err(error); + } self.emuplat_servicing.netvsp_state.push(save_state); Ok(()) diff --git a/openhcl/underhill_core/src/hardware_reseal.rs b/openhcl/underhill_core/src/hardware_reseal.rs index 1fbb8dbf98b..51fe81346f7 100644 --- a/openhcl/underhill_core/src/hardware_reseal.rs +++ b/openhcl/underhill_core/src/hardware_reseal.rs @@ -402,7 +402,7 @@ impl inspect::InspectMut for HardwareReseal { } impl StateUnit for HardwareReseal { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { // Starting or resuming does not create work or reset retry backoff. // Pending notifications and recovery survive a normal stop/start. self.schedule.running = true; @@ -413,6 +413,7 @@ impl StateUnit for HardwareReseal { pending_notification = self.notification.pending.load(Ordering::Acquire), "hardware reseal worker started" ); + Ok(()) } async fn stop(&mut self) { diff --git a/openhcl/underhill_core/src/hardware_reseal/tests.rs b/openhcl/underhill_core/src/hardware_reseal/tests.rs index 7081da1a42d..78ba94ca702 100644 --- a/openhcl/underhill_core/src/hardware_reseal/tests.rs +++ b/openhcl/underhill_core/src/hardware_reseal/tests.rs @@ -642,6 +642,7 @@ async fn finish_attempt( if start { drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); } drive_until(run.as_mut(), milestone).await; @@ -837,6 +838,7 @@ async fn stopped_worker_keeps_notification_and_start_honors_not_before(driver: D assert!(notification.pending.load(Ordering::SeqCst)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); assert!(!notification.pending.load(Ordering::SeqCst)); drive_until(run.as_mut(), send.call(StateRequest::Stop, ())) @@ -874,6 +876,7 @@ async fn stop_drains_blocked_write_and_final_flush_and_retains_late_event(driver let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), fixture.io.blocked()).await; let mut stop = pin!(send.call(StateRequest::Stop, ())); @@ -940,6 +943,7 @@ async fn blocked_hardware_keeps_executor_responsive_and_stop_drains_attempt(driv let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), gate.reached()).await; { @@ -1078,7 +1082,7 @@ async fn failed_report_keeps_recovery_pending_and_retries_successfully(driver: D let deadline = fixture.worker.schedule.deadline; let not_before = fixture.worker.schedule.not_before; - fixture.worker.start().await; + fixture.worker.start().await.unwrap(); fixture.worker.stop().await; assert!(fixture.worker.schedule.force_reseal); assert_eq!(fixture.worker.schedule.failures, 1); @@ -1300,6 +1304,7 @@ async fn changed_report_during_pre_or_post_verification_keeps_recovery_pending( let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); if post_flush { drive_until(run.as_mut(), fixture.io.blocked()).await; @@ -1418,6 +1423,7 @@ async fn migration_during_event_flush_is_detected_without_another_event(driver: let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), fixture.io.blocked()).await; fixture.hardware.identity.store(0x73, Ordering::SeqCst); @@ -1500,7 +1506,7 @@ async fn restore_and_reset_preserve_upgrade_and_failed_flush_obligation(driver: assert_eq!(fixture.worker.schedule.deadline, deadline); assert_eq!(fixture.worker.schedule.not_before, not_before); assert!(!fixture.worker.notification.pending.load(Ordering::SeqCst)); - fixture.worker.start().await; + fixture.worker.start().await.unwrap(); fixture.worker.stop().await; assert!(fixture.worker.schedule.force_reseal); assert_eq!(fixture.worker.schedule.failures, 1); @@ -1552,6 +1558,7 @@ async fn failed_final_flush_retries_write_even_when_cached_protector_matches( let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), fixture.io.blocked()).await; let stop = send.call(StateRequest::Stop, ()); @@ -1771,7 +1778,7 @@ async fn running_save_and_restore_fail_without_consuming_pending_work(driver: De let mut fixture = Fixture::new(&driver, true).await; fixture.worker.notification.notify(); let saved = fixture.worker.save().await.unwrap().unwrap(); - fixture.worker.start().await; + fixture.worker.start().await.unwrap(); let before = worker_snapshot(&fixture.worker); assert!(fixture.worker.save().await.is_err()); assert!(fixture.worker.restore(saved).await.is_err()); diff --git a/openhcl/underhill_core/src/lib.rs b/openhcl/underhill_core/src/lib.rs index 329d1c119f8..69cb928ad5c 100644 --- a/openhcl/underhill_core/src/lib.rs +++ b/openhcl/underhill_core/src/lib.rs @@ -686,11 +686,18 @@ async fn run_control( continue; }; - let was_resumed = workers + let was_resumed = match workers .vm_rpc .call(UhVmRpc::Resume, ()) .await - .context("failed to resumed VM")?; + .context("failed to resume VM")? + { + Ok(was_resumed) => was_resumed, + Err(error) => { + rpc.complete(Err(error)); + continue; + } + }; let was_halted = workers .vm_rpc diff --git a/openhcl/underhill_core/src/vmbus_relay_unit.rs b/openhcl/underhill_core/src/vmbus_relay_unit.rs index bce72bc9b17..21a172e45d9 100644 --- a/openhcl/underhill_core/src/vmbus_relay_unit.rs +++ b/openhcl/underhill_core/src/vmbus_relay_unit.rs @@ -43,8 +43,9 @@ impl VmbusRelayHandle { struct VmbusRelayUnit(HostVmbusTransport); impl StateUnit for &'_ VmbusRelayUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { diff --git a/openhcl/underhill_core/src/worker.rs b/openhcl/underhill_core/src/worker.rs index 702894bd524..694653db74e 100644 --- a/openhcl/underhill_core/src/worker.rs +++ b/openhcl/underhill_core/src/worker.rs @@ -465,6 +465,7 @@ impl Worker for UnderhillVmWorker { )); tracing::info!(CVM_ALLOWED, "terminating worker"); self.get_thread.join().unwrap(); + let state = state?; if let Some(state) = state { let params = UnderhillWorkerParameters { env_cfg: self.env_cfg, diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index ce8e17f1d79..2ae6541952f 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3697,7 +3697,12 @@ impl LoadedVm { if self.running { return false; } - self.state_units.start().await; + // Resume cannot report a failure to its callers, so treat a unit that + // fails to start like any other unrecoverable state unit failure. + self.state_units + .start() + .await + .expect("state units failed to start"); self.running = true; true } @@ -3738,7 +3743,10 @@ impl LoadedVm { let stop_guard = self.inner.partition_unit.temporarily_stop_vps().await; // Start state units so device config space is accessible. - self.state_units.start().await; + self.state_units + .start() + .await + .context("failed to start devices for PCI resource assignment")?; let result = ecam_config_access::assign_pci_resources_for_root_complexes( &self.inner.chipset, @@ -3824,7 +3832,12 @@ impl LoadedVm { } Err(err) => { if stopped { - self.state_units.start().await; + if let Err(start_error) = self.state_units.start().await { + rpc.complete(Err(RemoteError::new(start_error.context( + "worker restart failed and the VM could not resume", + )))); + break; + } } rpc.complete(Err(RemoteError::new(err))); } @@ -3897,7 +3910,15 @@ impl LoadedVm { ) .await?; self.inner.vmbus_devices.push(device); - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let device = self + .inner + .vmbus_devices + .pop() + .expect("device was just pushed"); + drop(device.remove().await); + return Err(error); + } anyhow::Ok(()) }) .await @@ -4043,7 +4064,15 @@ impl LoadedVm { // MSI. The guest may begin probing config space // immediately after receiving the interrupt, so // the device must be ready first. - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let (_, unit, _) = self + .inner + .pcie_hotplug_devices + .pop() + .expect("device was just pushed"); + unit.remove().await; + return Err(error); + } // Now attach the device and notify the guest. if let Err(e) = rc.lock().hotplug_add_device( diff --git a/vm/chipset_device_resources/Cargo.toml b/vm/chipset_device_resources/Cargo.toml index 4421fb5cc72..f8b19413036 100644 --- a/vm/chipset_device_resources/Cargo.toml +++ b/vm/chipset_device_resources/Cargo.toml @@ -16,6 +16,7 @@ vm_resource.workspace = true inspect.workspace = true async-trait.workspace = true +anyhow.workspace = true [lints] workspace = true diff --git a/vm/chipset_device_resources/src/lib.rs b/vm/chipset_device_resources/src/lib.rs index e2d6dc07aa4..8415c1c18f6 100644 --- a/vm/chipset_device_resources/src/lib.rs +++ b/vm/chipset_device_resources/src/lib.rs @@ -92,6 +92,7 @@ pub trait ConfigureChipsetDevice: Send { #[async_trait] trait DynChipsetDevice: ChipsetDevice + AsyncProtobufSaveRestore + InspectMut { fn start(&mut self); + async fn start_fallible(&mut self) -> anyhow::Result<()>; async fn stop(&mut self); async fn reset(&mut self); } @@ -101,7 +102,10 @@ impl anyhow::Result<()> { + ChangeDeviceState::start_fallible(self).await } async fn stop(&mut self) { self.stop().await @@ -132,6 +136,10 @@ impl ChangeDeviceState for ErasedChipsetDevice { self.0.start() } + async fn start_fallible(&mut self) -> anyhow::Result<()> { + self.0.start_fallible().await + } + async fn stop(&mut self) { self.0.stop().await } diff --git a/vm/devices/pci/vpci_relay/src/lib.rs b/vm/devices/pci/vpci_relay/src/lib.rs index 22e971f1f90..76bae759c18 100644 --- a/vm/devices/pci/vpci_relay/src/lib.rs +++ b/vm/devices/pci/vpci_relay/src/lib.rs @@ -490,7 +490,7 @@ impl VpciRelay { .await? }; - entry.insert(RelayedDevice { + let device = RelayedDevice { bus_instance_id: instance_id, bus_client: vpci_client, vpci_device: vpci_device.clone(), @@ -498,9 +498,13 @@ impl VpciRelay { bus_unit, device_unit, ready_to_remove: false, - }); + }; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + device.remove().await; + return Err(error); + } + entry.insert(device); Ok(()) } } diff --git a/vm/devices/virtio/virtio/src/tests.rs b/vm/devices/virtio/virtio/src/tests.rs index fed9bbb6eaf..18531a2bd07 100644 --- a/vm/devices/virtio/virtio/src/tests.rs +++ b/vm/devices/virtio/virtio/src/tests.rs @@ -1307,6 +1307,7 @@ type TestDeviceQueueWorkFn = #[inspect(skip)] struct FailingTestDevice { traits: DeviceTraits, + supports_save_restore: bool, } impl VirtioDevice for FailingTestDevice { @@ -1333,6 +1334,10 @@ impl VirtioDevice for FailingTestDevice { async fn stop_queue(&mut self, _idx: u16) -> Option { None } + + fn supports_save_restore(&self) -> bool { + self.supports_save_restore + } } #[derive(InspectMut)] @@ -3413,6 +3418,7 @@ async fn verify_enable_failure_mmio_does_not_set_driver_ok(_driver: DefaultDrive device_register_length: 0, ..Default::default() }, + supports_save_restore: false, }), &_driver, GuestMemory::empty(), @@ -3466,6 +3472,7 @@ async fn verify_enable_failure_pci_does_not_set_driver_ok(_driver: DefaultDriver device_register_length: 12, ..Default::default() }, + supports_save_restore: false, }), &_driver, GuestMemory::empty(), @@ -4838,6 +4845,114 @@ async fn mmio_save_restore_round_trip(driver: DefaultDriver) { dev2.stop().await; } +#[async_test] +async fn pci_fallible_start_reports_queue_restore_failure(driver: DefaultDriver) { + use vmcore::device_state::ChangeDeviceState; + use vmcore::save_restore::AsyncSaveRestore; + + let test_mem = VirtioTestMemoryAccess::new(); + let guest = VirtioTestGuest::new_split(&driver, &test_mem, 1, 4, true); + let mut dev = VirtioPciTestDevice::new(&driver, 1, &test_mem, None); + guest + .setup_pci_device(&mut dev, guest.queue_features()) + .await; + dev.pci_device.stop().await; + let saved = dev.pci_device.save().await.unwrap(); + + let doorbell_registration: Arc = test_mem.clone(); + let mem = GuestMemory::new("test", test_mem); + let msi_conn = MsiConnection::new(); + let mut restored = VirtioPciDevice::new( + Box::new(FailingTestDevice { + traits: DeviceTraits { + device_id: VirtioDeviceType::CONSOLE, + device_features: VirtioDeviceFeatures::new() + .with_bank(0, 2 | VIRTIO_F_RING_INDIRECT_DESC | VIRTIO_F_RING_EVENT_IDX) + .with_bank(1, VIRTIO_F_RING_PACKED), + max_queues: 1, + device_register_length: 12, + ..Default::default() + }, + supports_save_restore: true, + }), + &driver, + mem, + PciInterruptModel::Msix(&msi_conn.target()), + Some(doorbell_registration), + &mut ExternallyManagedMmioIntercepts, + None, + ) + .unwrap(); + restored.restore(saved).await.unwrap(); + + let error = restored.start_fallible().await.unwrap_err(); + assert!(format!("{error:#}").contains("intentional enable failure")); +} + +#[async_test] +async fn mmio_fallible_start_reports_queue_restore_failure(driver: DefaultDriver) { + use vmcore::device_state::ChangeDeviceState; + use vmcore::save_restore::SaveRestore; + + let test_mem = VirtioTestMemoryAccess::new(); + let doorbell_registration: Arc = test_mem.clone(); + let mem = GuestMemory::new("test", test_mem.clone()); + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let guest = VirtioTestGuest::new_split(&driver, &test_mem, 1, 4, true); + + let mut dev = VirtioMmioDevice::new( + Box::new(TestDevice::new( + &driver_source, + DeviceTraits { + device_id: VirtioDeviceType::CONSOLE, + device_features: VirtioDeviceFeatures::new() + .with_bank(0, 2 | VIRTIO_F_RING_INDIRECT_DESC | VIRTIO_F_RING_EVENT_IDX), + max_queues: 1, + device_register_length: 0, + ..Default::default() + }, + None, + )), + &driver_source.simple(), + mem.clone(), + LineInterrupt::detached(), + Some(doorbell_registration.clone()), + 0, + 1, + ) + .unwrap(); + guest + .setup_chipset_device(&mut dev, guest.queue_features()) + .await; + dev.stop().await; + let saved = dev.save().unwrap(); + + let mut restored = VirtioMmioDevice::new( + Box::new(FailingTestDevice { + traits: DeviceTraits { + device_id: VirtioDeviceType::CONSOLE, + device_features: VirtioDeviceFeatures::new() + .with_bank(0, 2 | VIRTIO_F_RING_INDIRECT_DESC | VIRTIO_F_RING_EVENT_IDX), + max_queues: 1, + device_register_length: 0, + ..Default::default() + }, + supports_save_restore: true, + }), + &driver_source.simple(), + mem, + LineInterrupt::detached(), + Some(doorbell_registration), + 0, + 1, + ) + .unwrap(); + restored.restore(saved).unwrap(); + + let error = restored.start_fallible().await.unwrap_err(); + assert!(format!("{error:#}").contains("intentional enable failure")); +} + #[async_test] async fn pci_save_restore_incompatible_features(driver: DefaultDriver) { use vmcore::device_state::ChangeDeviceState; @@ -4897,7 +5012,7 @@ async fn pci_save_not_supported_device(_driver: DefaultDriver) { let msi_conn = MsiConnection::new(); - // FailingTestDevice does not override supports_save_restore (default false). + // Configure the failing device without save/restore support. let mut dev = VirtioPciDevice::new( Box::new(FailingTestDevice { traits: DeviceTraits { @@ -4907,6 +5022,7 @@ async fn pci_save_not_supported_device(_driver: DefaultDriver) { device_register_length: 0, ..Default::default() }, + supports_save_restore: false, }), &_driver, GuestMemory::empty(), @@ -4936,6 +5052,7 @@ async fn mmio_save_not_supported_device(_driver: DefaultDriver) { device_register_length: 0, ..Default::default() }, + supports_save_restore: false, }), &_driver, GuestMemory::empty(), diff --git a/vm/devices/virtio/virtio/src/transport/core.rs b/vm/devices/virtio/virtio/src/transport/core.rs index 0ef24ccc726..14b7d35d573 100644 --- a/vm/devices/virtio/virtio/src/transport/core.rs +++ b/vm/devices/virtio/virtio/src/transport/core.rs @@ -356,49 +356,64 @@ impl VirtioTransportCore { } } - /// `ChangeDeviceState::start()` implementation. - pub fn start(&mut self, ops: &mut dyn TransportOps) { - if self.device_status.driver_ok() { - let features = self.driver_feature; - let mut queues = Vec::new(); - for (i, qd) in self.queues.iter_mut().enumerate() { - if !qd.params.enable { - continue; - } - let initial_state = qd.saved_state.take(); - queues.push(( - i, - qd.params, - qd.msix_vector, - qd.event.clone(), - initial_state, - )); + fn start_params(&mut self, ops: &mut dyn TransportOps) -> Option { + if !self.device_status.driver_ok() { + return None; + } + + let features = self.driver_feature; + let mut queues = Vec::new(); + for (i, qd) in self.queues.iter_mut().enumerate() { + if !qd.params.enable { + continue; } - let queues: Vec<_> = queues - .into_iter() - .map(|(i, params, msix_vector, event, initial_state)| { - let notify = ops.create_queue_interrupt(i, msix_vector); - ( - i as u16, - QueueResources { - params, - notify, - event, - guest_memory: self.guest_memory.clone(), - }, - initial_state, - ) - }) - .collect(); + let initial_state = qd.saved_state.take(); + queues.push(( + i, + qd.params, + qd.msix_vector, + qd.event.clone(), + initial_state, + )); + } + let queues = queues + .into_iter() + .map(|(i, params, msix_vector, event, initial_state)| { + let notify = ops.create_queue_interrupt(i, msix_vector); + ( + i as u16, + QueueResources { + params, + notify, + event, + guest_memory: self.guest_memory.clone(), + }, + initial_state, + ) + }) + .collect(); - let params = StartParams { queues, features }; + Some(StartParams { queues, features }) + } - // Fire and forget — start() is sync, can't await. + /// `ChangeDeviceState::start()` implementation. + pub fn start(&mut self, ops: &mut dyn TransportOps) { + if let Some(params) = self.start_params(ops) { self.device_sender .send(DeviceCommand::Start(Rpc::detached(params))); } } + /// `ChangeDeviceState::start_fallible()` implementation. + pub async fn start_fallible(&mut self, ops: &mut dyn TransportOps) -> anyhow::Result<()> { + if let Some(params) = self.start_params(ops) { + self.device_sender + .call_failable(DeviceCommand::Start, params) + .await?; + } + Ok(()) + } + /// `ChangeDeviceState::stop()` implementation. pub async fn stop(&mut self, ops: &mut dyn TransportOps) { if let Some(result) = self.state.drain().await { diff --git a/vm/devices/virtio/virtio/src/transport/mmio.rs b/vm/devices/virtio/virtio/src/transport/mmio.rs index b418e8209e7..11a1c11769a 100644 --- a/vm/devices/virtio/virtio/src/transport/mmio.rs +++ b/vm/devices/virtio/virtio/src/transport/mmio.rs @@ -359,6 +359,10 @@ impl ChangeDeviceState for VirtioMmioDevice { self.core.start(&mut self.mmio); } + async fn start_fallible(&mut self) -> anyhow::Result<()> { + self.core.start_fallible(&mut self.mmio).await + } + async fn stop(&mut self) { self.core.stop(&mut self.mmio).await; } diff --git a/vm/devices/virtio/virtio/src/transport/pci.rs b/vm/devices/virtio/virtio/src/transport/pci.rs index 22a987d64a0..5659c467620 100644 --- a/vm/devices/virtio/virtio/src/transport/pci.rs +++ b/vm/devices/virtio/virtio/src/transport/pci.rs @@ -776,6 +776,10 @@ impl ChangeDeviceState for VirtioPciDevice { self.core.start(&mut self.pci); } + async fn start_fallible(&mut self) -> anyhow::Result<()> { + self.core.start_fallible(&mut self.pci).await + } + async fn stop(&mut self) { self.core.stop(&mut self.pci).await; } diff --git a/vm/devices/virtio/virtio/src/transport/task.rs b/vm/devices/virtio/virtio/src/transport/task.rs index 79cb453b018..9ab01549879 100644 --- a/vm/devices/virtio/virtio/src/transport/task.rs +++ b/vm/devices/virtio/virtio/src/transport/task.rs @@ -273,11 +273,6 @@ pub async fn run_device_task( rpc.handle(async |()| task.stop().await).await; } DeviceCommand::Start(rpc) => { - // Start is used by ChangeDeviceState::start(), which is - // sync and uses Rpc::detached() — errors are logged here - // but not propagated to the transport. - // TODO: update ChangeDeviceState to allow async start() - // so failures can be handled by the transport. rpc.handle_failable(async |params| task.start(params).await) .await; } diff --git a/vm/vmcore/src/device_state.rs b/vm/vmcore/src/device_state.rs index bffa5515366..cdf0b18e1a4 100644 --- a/vm/vmcore/src/device_state.rs +++ b/vm/vmcore/src/device_state.rs @@ -27,14 +27,30 @@ pub trait ChangeDeviceState { // to finish, which may require a bunch of other changes. fn start(&mut self); + /// Starts a device and waits until startup has either completed or failed. + /// + /// The default preserves the notification-only behavior of [`Self::start`]. + /// Devices that restore process-local workers should override this so a + /// restore failure is reported before dependent units start. + /// + /// After this is called, [`Self::stop`] may be called to roll back startup, + /// regardless of whether this method succeeds. Implementations that return + /// an error must leave the device in a state where `stop` is safe and + /// cleans up any partial startup. + fn start_fallible(&mut self) -> impl Send + Future> { + self.start(); + async { Ok(()) } + } + /// Stops a device's asynchronous work. /// /// After this returns, the device must not process any additional work. It /// should be in a stable state where it can be saved without losing data /// (if it implements the appropriate trait). /// - /// Callers must ensure that the device is in a started state before calling - /// this method. + /// Callers must ensure that the device has had [`Self::start`] or + /// [`Self::start_fallible`] called and has not subsequently been stopped. + /// This includes a `start_fallible` call that returned an error. fn stop(&mut self) -> impl Send + Future; /// Resets the device state to its initial state, for a fresh boot. diff --git a/vmm_core/src/device_builder.rs b/vmm_core/src/device_builder.rs index 089ac1444c9..440d98a9dc4 100644 --- a/vmm_core/src/device_builder.rs +++ b/vmm_core/src/device_builder.rs @@ -144,7 +144,11 @@ pub async fn build_dynamic_vpci_device( }; let pending_offer = pending_offer.context("missing deferred VPCI channel offer")?; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + vpci_unit.remove().await; + pci_unit.remove().await; + return Err(error); + } if let Err(error) = pending_offer .offer_registered(&vpci_bus, driver_source, vmbus, state_units.is_running()) .await diff --git a/vmm_core/src/input_distributor.rs b/vmm_core/src/input_distributor.rs index 6aafd50d85d..7d6aef61e1c 100644 --- a/vmm_core/src/input_distributor.rs +++ b/vmm_core/src/input_distributor.rs @@ -195,8 +195,9 @@ struct Inner { } impl StateUnit for Inner { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.running = true; + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/src/partition_unit.rs b/vmm_core/src/partition_unit.rs index 3dd1ea5f802..962d9db6dab 100644 --- a/vmm_core/src/partition_unit.rs +++ b/vmm_core/src/partition_unit.rs @@ -594,10 +594,11 @@ impl Drop for StopGuard { } impl StateUnit for PartitionUnitRunner { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.partition.thaw_time(); self.unit_started = true; self.try_start(); + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/src/partition_unit/tests.rs b/vmm_core/src/partition_unit/tests.rs index 2c62663243c..29e0690ec37 100644 --- a/vmm_core/src/partition_unit/tests.rs +++ b/vmm_core/src/partition_unit/tests.rs @@ -131,7 +131,7 @@ fn time_stays_frozen_across_initial_restore_and_reset() { StateUnit::reset(&mut runner).await.unwrap(); assert!(clock.lock().frozen); assert!(clock.lock().transitions.is_empty()); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); assert_eq!(clock.lock().transitions, [false]); }); } @@ -141,7 +141,7 @@ fn full_stop_freezes_time_until_resume() { block_on(async { let (mut runner, clock, _) = new_runner(); for _ in 0..2 { - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); assert!(!clock.lock().frozen); StateUnit::stop(&mut runner).await; let state = StateUnit::save(&mut runner).await.unwrap().unwrap(); @@ -157,7 +157,7 @@ fn full_stop_freezes_time_until_resume() { fn temporary_stops_and_debugger_halts_leave_time_running() { block_on(async { let (mut runner, clock, halt) = new_runner(); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); runner.stop_vps().await; runner.stop_vps().await; assert!(!clock.lock().frozen); @@ -178,14 +178,14 @@ fn temporary_stops_and_debugger_halts_leave_time_running() { fn temporary_stop_guards_do_not_override_full_stop() { block_on(async { let (mut runner, clock, _) = new_runner(); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); runner.stop_vps().await; runner.stop_vps().await; StateUnit::stop(&mut runner).await; runner.resume_vps(); assert!(clock.lock().frozen); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); assert!(!clock.lock().frozen); runner.resume_vps(); assert_eq!(clock.lock().transitions, [false, true, false]); @@ -196,7 +196,7 @@ fn temporary_stop_guards_do_not_override_full_stop() { fn temporary_scrub_thaws_time_before_restarting_vps() { block_on(async { let (mut runner, clock, _) = new_runner(); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); runner.stop_vps().await; runner.partition.scrub_vtl(Vtl::Vtl2).unwrap(); assert!(clock.lock().frozen); diff --git a/vmm_core/src/vmbus_unit.rs b/vmm_core/src/vmbus_unit.rs index 5d6d9b0f398..76024683b0d 100644 --- a/vmm_core/src/vmbus_unit.rs +++ b/vmm_core/src/vmbus_unit.rs @@ -78,8 +78,9 @@ impl VmbusServerHandle { struct VmbusServerUnit(VmbusServer); impl StateUnit for &'_ VmbusServerUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { @@ -134,8 +135,9 @@ impl ChannelUnit { } impl StateUnit for &'_ ChannelUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { @@ -205,8 +207,9 @@ impl Inspect for SimpleChannelUnit { } impl StateUnit for &'_ SimpleChannelUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/src/vmtime_unit.rs b/vmm_core/src/vmtime_unit.rs index a53688cb527..9fa55a3c138 100644 --- a/vmm_core/src/vmtime_unit.rs +++ b/vmm_core/src/vmtime_unit.rs @@ -17,8 +17,9 @@ use vmcore::vmtime::VmTimeKeeper; struct KeeperUnit<'a>(#[inspect(mut)] &'a mut VmTimeKeeper); impl StateUnit for KeeperUnit<'_> { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start().await; + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 87a9525e73d..928730e6a39 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -30,6 +30,8 @@ #![forbid(unsafe_code)] +mod start; + use futures::FutureExt; use futures::StreamExt; use futures::future::join_all; @@ -70,7 +72,7 @@ use vmcore::save_restore::SavedStateBlob; #[derive(Debug, MeshPayload)] pub enum StateRequest { /// Start asynchronous operations. - Start(Rpc<(), ()>), + Start(FailableRpc<(), ()>), /// Stop asynchronous operations. Stop(Rpc<(), ()>), @@ -96,7 +98,7 @@ pub enum StateRequest { #[expect(async_fn_in_trait)] // Don't need Send bounds pub trait StateUnit: InspectMut { /// Start asynchronous processing. - async fn start(&mut self); + async fn start(&mut self) -> anyhow::Result<()>; /// Stop asynchronous processing. async fn stop(&mut self); @@ -192,7 +194,7 @@ impl StateRequest { /// Runs this state request against `unit`. pub async fn apply(self, unit: &mut impl StateUnit) { match self { - StateRequest::Start(rpc) => rpc.handle(async |()| unit.start().await).await, + StateRequest::Start(rpc) => rpc.handle_failable(async |()| unit.start().await).await, StateRequest::Stop(rpc) => rpc.handle(async |()| unit.stop().await).await, StateRequest::Reset(rpc) => rpc.handle_failable(async |()| unit.reset().await).await, StateRequest::Save(rpc) => rpc.handle_failable(async |()| unit.save().await).await, @@ -222,6 +224,8 @@ enum State { Resetting, Saving, Restoring, + StartUncertain, + StartFailedStopped, } #[derive(Debug)] @@ -486,36 +490,69 @@ impl StateUnits { self.running } + fn check_no_terminal_units(&self, op: &'static str) -> Result<(), StateTransitionError> { + let inner = self.inner.lock(); + check( + op, + inner + .units + .values() + .filter(|unit| { + matches!( + unit.state, + State::StartUncertain | State::StartFailedStopped + ) + }) + .map(|unit| { + ( + unit.name.clone(), + Err::<(), _>(anyhow::anyhow!( + "unit is in terminal state {:?}", + unit.state + )), + ) + }), + ) + } + /// Starts any units that are individually stopped, because they were added /// via [`StateUnits::add`] while the VM was running. /// /// Does nothing if all units are stopped, via [`StateUnits::stop`]. - pub async fn start_stopped_units(&mut self) { + pub async fn start_stopped_units(&mut self) -> anyhow::Result<()> { if self.is_running() { - self.start().await; + self.start().await?; } + Ok(()) } /// Starts all the state units. - pub async fn start(&mut self) { + pub async fn start(&mut self) -> anyhow::Result<()> { self.restore_allowed = false; - self.run_op( - "start", - None, - State::Stopped, - State::Starting, - State::Running, - StateRequest::Start, - |_, _| Some(()), - |unit| Some(&unit.dependencies), - ) - .await; - self.running = true; + self.start_with_rollback().await } /// Stops all the state units. pub async fn stop(&mut self) { assert!(self.running); + // Make uncertain starts receive Stop and already-cleaned failures skip + // it, then restore both terminal markers after the ordered operation. + let terminal_states = { + let mut inner = self.inner.lock(); + inner + .units + .iter_mut() + .filter_map(|(&id, unit)| { + let state = unit.state; + match state { + State::StartUncertain => unit.state = State::Running, + State::StartFailedStopped => unit.state = State::Stopped, + _ => return None, + } + Some((id, state)) + }) + .collect::>() + }; // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( @@ -529,6 +566,13 @@ impl StateUnits { |unit| Some(&unit.dependents), ) .await; + let mut inner = self.inner.lock(); + for (id, state) in terminal_states { + if let Some(unit) = inner.units.get_mut(&id) { + assert_eq!(unit.state, State::Stopped); + unit.state = state; + } + } self.running = false; } @@ -537,6 +581,7 @@ impl StateUnits { /// Panics if running. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("reset")?; self.restore_allowed = false; // Reset in dependency order so that dependants observe their // dependencies' reset state. @@ -563,6 +608,7 @@ impl StateUnits { /// Panics if running. pub async fn save(&mut self) -> Result, StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("save")?; // Save can occur in any order since it will not observably mutate // state. let r = self @@ -601,6 +647,7 @@ impl StateUnits { states: Vec, ) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("restore")?; if !self.restore_allowed { return Err(StateTransitionError { op: "restore", @@ -801,7 +848,7 @@ impl Inner { let units = if let Some(unit_ids) = unit_ids { unit_ids .iter() - .map(|id| map(*id, &self.units[id])) + .filter_map(|id| self.units.get(id).map(|unit| map(*id, unit))) .collect() } else { self.units.iter().map(|(id, unit)| map(*id, unit)).collect() @@ -1047,6 +1094,8 @@ impl Ready { #[cfg(test)] mod tests { + mod start; + use super::StateTransitionErrorKind; use super::StateUnit; use super::StateUnits; @@ -1079,7 +1128,9 @@ mod tests { struct SavedState(bool); impl StateUnit for TestUnit { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} @@ -1121,7 +1172,9 @@ mod tests { } impl StateUnit for TestUnitSetDep { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} @@ -1161,7 +1214,9 @@ mod tests { } impl StateUnit for FailingUnit { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} async fn reset(&mut self) -> anyhow::Result<()> { @@ -1219,7 +1274,7 @@ mod tests { let state = units.save().await.unwrap(); units.restore(state).await.unwrap(); assert_eq!(calls.load(Ordering::Relaxed), 2); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); assert!(units.restore(state).await.is_err()); @@ -1293,13 +1348,13 @@ mod tests { .add("b") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) .unwrap(); - units.start().await; + units.start().await.unwrap(); let _c = units .add("c") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)); units.stop().await; - units.start().await; + units.start().await.unwrap(); units.stop().await; @@ -1347,7 +1402,7 @@ mod tests { ) }) .unwrap(); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); @@ -1457,7 +1512,7 @@ mod tests { ); } - units.start().await; + units.start().await.unwrap(); units.stop().await; a.remove().await; @@ -1467,7 +1522,7 @@ mod tests { assert!(inner.units[&c.handle().id.id].dependencies.is_empty()); } - units.start().await; + units.start().await.unwrap(); units.stop().await; // Removing the remaining unit leaves the set empty. diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs new file mode 100644 index 00000000000..217802fc89a --- /dev/null +++ b/vmm_core/state_unit/src/start.rs @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Fallible start with rollback. If a unit fails to start, its dependents do +//! not start, and the units that did start are stopped again. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use std::time::Instant; + +impl StateUnits { + /// Starts all stopped units in dependency order. + /// + /// A unit does not start if one of its dependencies fails to start. On any + /// failure, the units that did start are stopped again, and a unit whose + /// start failed or is uncertain is left in a terminal state that fails + /// later starts. + pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { + enum StartResult { + Started, + DependencyFailed, + Failed(anyhow::Error), + Uncertain(anyhow::Error), + } + + let was_running = self.running; + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + if let Some(unit) = inner + .units + .values() + .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) + { + anyhow::bail!( + "state unit '{}' cannot be started from terminal state {:?}", + unit.name, + unit.state + ); + } + ready_set = inner.ready_set(None); + for (&id, unit) in &mut inner.units { + if unit.state == State::Running { + ready_set.done(id, true); + continue; + } + assert_eq!( + unit.state, + State::Stopped, + "unit {} is not stopped before start", + unit.name, + ); + let name = unit.name.clone(); + let dependencies = unit.dependencies.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + operations.push(async move { + if !ready_set.wait("start", id, &dependencies).await { + ready_set.done(id, false); + return (name, id, StartResult::DependencyFailed); + } + + let result = match start.await { + Ok(Some(Ok(()))) => StartResult::Started, + Ok(Some(Err(error))) => StartResult::Failed(error.into()), + Ok(None) => { + StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) + } + Err(error) => StartResult::Uncertain(error.into()), + }; + ready_set.done(id, matches!(result, StartResult::Started)); + (name, id, result) + }); + unit.state = State::Starting; + } + } + + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "state-unit start complete"); + + let mut failures = Vec::new(); + let mut started_ids = Vec::new(); + let mut failed_or_uncertain_ids = Vec::new(); + { + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + match result { + StartResult::Started => { + unit.state = State::Running; + started_ids.push(id); + } + StartResult::DependencyFailed => { + unit.state = State::Stopped; + failures.push(format!("{name}: a dependency did not start")); + } + StartResult::Failed(error) => { + unit.state = State::StartUncertain; + failed_or_uncertain_ids.push(id); + failures.push(format!("{name}: {error:#}")); + } + StartResult::Uncertain(error) => { + unit.state = State::StartUncertain; + failed_or_uncertain_ids.push(id); + failures.push(format!("{name}: {error:#}")); + } + } + } + } + + if failures.is_empty() { + self.running = true; + Ok(()) + } else { + failures.extend( + self.stop_after_failed_start( + "failed_start_cleanup", + &failed_or_uncertain_ids, + State::StartUncertain, + State::StartFailedStopped, + ) + .await, + ); + failures.extend( + self.stop_after_failed_start( + "failed_start_rollback", + &started_ids, + State::Running, + State::Stopped, + ) + .await, + ); + self.running = was_running; + anyhow::bail!("state units could not start: {}", failures.join("; ")) + } + } + + async fn stop_after_failed_start( + &self, + op: &'static str, + unit_ids: &[u64], + old_state: State, + stopped_state: State, + ) -> Vec { + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + ready_set = inner.ready_set(Some(unit_ids)); + for (&id, unit) in inner + .units + .iter_mut() + .filter(|(id, _)| ready_set.0.contains_key(id)) + { + assert_eq!(unit.state, old_state); + let name = unit.name.clone(); + let dependents = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); + operations.push(async move { + ready_set.wait(op, id, &dependents).await; + let result = stop.await; + ready_set.done(id, true); + (name, id, result) + }); + unit.state = State::Stopping; + } + } + + let results = join_all(operations).await; + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + match result { + Ok(Some(())) => unit.state = stopped_state, + Ok(None) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {op} stop request was not sent")); + } + Err(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {op} stop failed: {error:#}")); + } + } + } + failures + } +} diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs new file mode 100644 index 00000000000..da37724a75a --- /dev/null +++ b/vmm_core/state_unit/src/tests/start.rs @@ -0,0 +1,346 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for fallible start. + +use crate::StateRequest; +use crate::StateUnit; +use crate::StateUnits; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use test_with_tracing::test; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +struct StartResultUnit { + started: Arc, + stopped: Arc, + fail: bool, +} + +impl StateUnit for StartResultUnit { + async fn start(&mut self) -> anyhow::Result<()> { + self.started.store(true, Ordering::Relaxed); + anyhow::ensure!(!self.fail, "intentional start failure"); + Ok(()) + } + + async fn stop(&mut self) { + assert!(!self.stopped.swap(true, Ordering::Relaxed)); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for StartResultUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +#[async_test] +async fn test_start_failure_blocks_dependent(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let independent_started = Arc::new(AtomicBool::new(false)); + let independent_stopped = Arc::new(AtomicBool::new(false)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: independent_started.clone(), + stopped: independent_stopped.clone(), + fail: false, + }, + recv, + ) + }) + .unwrap(); + let failed_started = Arc::new(AtomicBool::new(false)); + let failed_stopped = Arc::new(AtomicBool::new(false)); + let failed = units + .add("failed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: failed_started.clone(), + stopped: failed_stopped.clone(), + fail: true, + }, + recv, + ) + }) + .unwrap(); + let dependent_started = Arc::new(AtomicBool::new(false)); + let _dependent = units + .add("dependent") + .depends_on(failed.handle()) + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: dependent_started.clone(), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("intentional start failure")); + assert!(independent_started.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(failed_started.load(Ordering::Relaxed)); + assert!(failed_stopped.load(Ordering::Relaxed)); + assert!(!dependent_started.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_stop_cleans_up_failed_start_while_running(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _running = units + .add("running") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let failed_stopped = Arc::new(AtomicBool::new(false)); + let _failed = units + .add("failed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: failed_stopped.clone(), + fail: true, + }, + recv, + ) + }) + .unwrap(); + + units.start_stopped_units().await.unwrap_err(); + assert!(units.is_running()); + let retry_error = units.start_stopped_units().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + units.stop().await; + assert!(failed_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_uncertain_start_is_cleaned_once_and_remains_terminal(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _running = units + .add("running") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let uncertain_started = Arc::new(AtomicBool::new(false)); + let uncertain_stopped = Arc::new(AtomicBool::new(false)); + let stop_requests = Arc::new(AtomicUsize::new(0)); + let _uncertain = units + .add("uncertain") + .spawn(&driver, |mut recv| { + let stop_requests = stop_requests.clone(); + let mut unit = StartResultUnit { + started: uncertain_started.clone(), + stopped: uncertain_stopped.clone(), + fail: false, + }; + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.started.store(true, Ordering::Relaxed); + drop(response); + } + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + + units.start_stopped_units().await.unwrap_err(); + assert!(uncertain_started.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 1); + assert!(!uncertain_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + let retry_error = units.start_stopped_units().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + + units.stop().await; + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(uncertain_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_initial_uncertain_start_is_cleaned(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let uncertain_started = Arc::new(AtomicBool::new(false)); + let uncertain_stopped = Arc::new(AtomicBool::new(false)); + let _uncertain = units + .add("uncertain") + .spawn(&driver, |mut recv| { + let mut unit = StartResultUnit { + started: uncertain_started.clone(), + stopped: uncertain_stopped.clone(), + fail: false, + }; + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.started.store(true, Ordering::Relaxed); + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + + units.start().await.unwrap_err(); + assert!(uncertain_started.load(Ordering::Relaxed)); + assert!(uncertain_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_started_unit_exit_during_rollback_returns_error(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _exited = units + .add("exited") + .spawn(&driver, |mut recv| async move { + let unit = StartResultUnit { + started: Arc::new(AtomicBool::new(true)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }; + let request = recv.next().await.unwrap(); + let StateRequest::Start(response) = request else { + panic!("unexpected request"); + }; + response.handle_failable_sync(|()| anyhow::Ok(())); + unit + }) + .unwrap(); + let _failed = units + .add("failed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: true, + }, + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!( + error + .to_string() + .contains("failed_start_rollback stop failed") + ); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_rollback_ready_set_ignores_removed_unit(driver: DefaultDriver) { + let units = StateUnits::new(); + let unit = units + .add("removed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + let id = unit.handle().id.id; + unit.remove().await; + + let ready_set = units.inner.lock().ready_set(Some(&[id])); + assert!(ready_set.0.is_empty()); +} diff --git a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs index 5f296e7e28f..fe696dec6ab 100644 --- a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs +++ b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs @@ -35,7 +35,7 @@ impl InspectMut for ArcMutexChipsetDeviceUnit { /// Object-safe trait for the subset of [`VmmChipsetDevice`] that we use here. #[async_trait] trait DynDevice: InspectMut + Send { - fn start(&mut self); + async fn start(&mut self) -> anyhow::Result<()>; async fn stop(&mut self); async fn reset(&mut self); fn poll_device(&mut self, cx: &mut Context<'_>); @@ -45,8 +45,8 @@ trait DynDevice: InspectMut + Send { #[async_trait] impl DynDevice for T { - fn start(&mut self) { - self.start() + async fn start(&mut self) -> anyhow::Result<()> { + self.start_fallible().await } async fn stop(&mut self) { @@ -139,13 +139,14 @@ impl ArcMutexChipsetDeviceUnit { } impl StateUnit for ArcMutexChipsetDeviceUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { + self.device.clone().close().start().await?; self.running = true; - // Poll the device at least once. + // Poll the device at least once after startup completes. let mut device = self.device.lock(); - device.start(); device.poll_device(&mut Context::from_waker(&waker_ref(&self.poll_event))); + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/vmotherboard/src/chipset/builder/mod.rs b/vmm_core/vmotherboard/src/chipset/builder/mod.rs index 788e80f98b1..a8e47c0fae0 100644 --- a/vmm_core/vmotherboard/src/chipset/builder/mod.rs +++ b/vmm_core/vmotherboard/src/chipset/builder/mod.rs @@ -379,7 +379,9 @@ mod chipset_unit { pub struct ChipsetUnit<'a>(pub &'a Chipset); impl StateUnit for ChipsetUnit<'_> { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} diff --git a/vmm_core/vmotherboard/src/chipset/line_sets.rs b/vmm_core/vmotherboard/src/chipset/line_sets.rs index dd6d73f7d7b..9d8517a8176 100644 --- a/vmm_core/vmotherboard/src/chipset/line_sets.rs +++ b/vmm_core/vmotherboard/src/chipset/line_sets.rs @@ -60,7 +60,9 @@ impl LineSets { struct LineSetUnit<'a>(&'a LineSet); impl StateUnit for LineSetUnit<'_> { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} diff --git a/workers/chipset_device_worker/src/protocol.rs b/workers/chipset_device_worker/src/protocol.rs index 1039257b60a..1e200c0c823 100644 --- a/workers/chipset_device_worker/src/protocol.rs +++ b/workers/chipset_device_worker/src/protocol.rs @@ -60,7 +60,7 @@ pub(crate) enum DeviceRequest { /// Perform a PCI config space write. PciConfigWrite(WriteRequest, PciConfigByteEnable), /// Start the device - Start, + Start(mesh::rpc::FailableRpc<(), ()>), /// Stop the device Stop(Rpc<(), ()>), /// Reset the device diff --git a/workers/chipset_device_worker/src/proxy.rs b/workers/chipset_device_worker/src/proxy.rs index 495551391ee..eceb48f1da1 100644 --- a/workers/chipset_device_worker/src/proxy.rs +++ b/workers/chipset_device_worker/src/proxy.rs @@ -23,6 +23,7 @@ use chipset_device::pci::PciConfigSpace; use chipset_device::pio::PortIoIntercept; use chipset_device::poll_device::PollDevice; use chipset_device_resources::ResolveChipsetDeviceHandleParams; +use futures::FutureExt; use futures::StreamExt; use inspect::Inspect; use inspect::InspectMut; @@ -267,21 +268,38 @@ impl PollDevice for ChipsetDeviceProxy { impl ChangeDeviceState for ChipsetDeviceProxy { fn start(&mut self) { - self.req_send.send(DeviceRequest::Start) + self.req_send + .send(DeviceRequest::Start(mesh::rpc::Rpc::detached(()))) + } + + async fn start_fallible(&mut self) -> anyhow::Result<()> { + start_remote_device( + self.req_send.clone(), + &mut self.gm_proxy, + &mut self.enc_gm_proxy, + ) + .await?; + Ok(()) } async fn stop(&mut self) { - self.req_send - .call(DeviceRequest::Stop, ()) - .await - .expect("failed to stop remote device") + serve_guest_memory_until( + &mut self.gm_proxy, + &mut self.enc_gm_proxy, + self.req_send.call(DeviceRequest::Stop, ()), + ) + .await + .expect("failed to stop remote device") } async fn reset(&mut self) { - self.req_send - .call(DeviceRequest::Reset, ()) - .await - .expect("failed to reset remote device"); + serve_guest_memory_until( + &mut self.gm_proxy, + &mut self.enc_gm_proxy, + self.req_send.call(DeviceRequest::Reset, ()), + ) + .await + .expect("failed to reset remote device"); self.in_flight_reads.clear(); self.in_flight_writes.clear(); @@ -289,32 +307,75 @@ impl ChangeDeviceState for ChipsetDeviceProxy { } } +/// Waits for a response from the remote device while servicing its guest +/// memory requests. +/// +/// The remote device blocks on the parent for every guest memory access, so +/// the parent has to keep polling the proxies for as long as it waits on a +/// request that the device may serve by accessing guest memory, such as +/// starting or stopping its queues. +async fn serve_guest_memory_until( + gm_proxy: &mut GuestMemoryProxy, + enc_gm_proxy: &mut GuestMemoryProxy, + response: impl Future, +) -> T { + let mut response = std::pin::pin!(response); + std::future::poll_fn(|cx| { + gm_proxy.poll(cx); + enc_gm_proxy.poll(cx); + response.poll_unpin(cx) + }) + .await +} + +async fn start_remote_device( + req_send: mesh::Sender, + gm_proxy: &mut GuestMemoryProxy, + enc_gm_proxy: &mut GuestMemoryProxy, +) -> anyhow::Result<()> { + serve_guest_memory_until( + gm_proxy, + enc_gm_proxy, + req_send.call_failable(DeviceRequest::Start, ()), + ) + .await?; + Ok(()) +} + #[async_trait] impl AsyncProtobufSaveRestore for ChipsetDeviceProxy { async fn save(&mut self) -> Result { // TODO: Do we need to include any state from ourselves? - self.req_send - .call(DeviceRequest::Save, ()) - .await - .map_err(|e: mesh::rpc::RpcError| SaveError::Other(anyhow::anyhow!(e)))? - .map_err(|e: mesh::error::RemoteError| SaveError::Other(anyhow::anyhow!(e))) + serve_guest_memory_until( + &mut self.gm_proxy, + &mut self.enc_gm_proxy, + self.req_send.call(DeviceRequest::Save, ()), + ) + .await + .map_err(|e: mesh::rpc::RpcError| SaveError::Other(anyhow::anyhow!(e)))? + .map_err(|e: mesh::error::RemoteError| SaveError::Other(anyhow::anyhow!(e))) } async fn restore(&mut self, state: SavedStateBlob) -> Result<(), RestoreError> { - self.req_send - .call(DeviceRequest::Restore, state) - .await - .map_err(|e: mesh::rpc::RpcError| RestoreError::Other(anyhow::anyhow!(e)))? - .map_err(|e: mesh::error::RemoteError| RestoreError::Other(anyhow::anyhow!(e))) + serve_guest_memory_until( + &mut self.gm_proxy, + &mut self.enc_gm_proxy, + self.req_send.call(DeviceRequest::Restore, state), + ) + .await + .map_err(|e: mesh::rpc::RpcError| RestoreError::Other(anyhow::anyhow!(e)))? + .map_err(|e: mesh::error::RemoteError| RestoreError::Other(anyhow::anyhow!(e))) } } #[cfg(test)] mod tests { use super::ChipsetDeviceProxy; + use super::start_remote_device; use crate::RemoteDynamicResolvers; use crate::guestmem::GuestMemoryProxy; use crate::protocol::DeviceInit; + use crate::protocol::DeviceRequest; use crate::worker::RemoteChipsetDeviceHandleParams; use crate::worker::RemoteChipsetDeviceWorker; use crate::worker::RemoteChipsetDeviceWorkerParameters; @@ -333,6 +394,7 @@ mod tests { use test_with_tracing::test; use vm_resource::IntoResource; use vm_resource::ResolveResource; + use vm_resource::Resource; use vm_resource::ResourceId; use vm_resource::ResourceResolver; use vm_resource::kind::ChipsetDeviceHandleKind; @@ -437,16 +499,147 @@ mod tests { } } + /// A device that reads guest memory from every state change handler, so + /// the handlers only complete while the parent services the remote guest + /// memory proxies. Starting it reads guest memory and then fails. + #[derive(InspectMut)] + struct GuestMemoryDevice { + #[inspect(skip)] + guest_memory: GuestMemory, + #[inspect(skip)] + encrypted_guest_memory: GuestMemory, + } + + impl GuestMemoryDevice { + fn read_guest_memory(&self) -> anyhow::Result<()> { + let mut data = [0]; + self.guest_memory.read_at(0, &mut data)?; + anyhow::ensure!(data == [0x12], "unexpected guest memory"); + self.encrypted_guest_memory.read_at(0, &mut data)?; + anyhow::ensure!(data == [0x34], "unexpected encrypted guest memory"); + Ok(()) + } + } + + impl ChipsetDevice for GuestMemoryDevice {} + + impl ChangeDeviceState for GuestMemoryDevice { + fn start(&mut self) {} + + async fn start_fallible(&mut self) -> anyhow::Result<()> { + self.read_guest_memory()?; + anyhow::bail!("start failed") + } + + async fn stop(&mut self) { + self.read_guest_memory().unwrap(); + } + + async fn reset(&mut self) { + self.read_guest_memory().unwrap(); + } + } + + impl AsyncSaveRestore for GuestMemoryDevice { + type SavedState = TestSavedState; + + async fn save(&mut self) -> Result { + self.read_guest_memory().map_err(SaveError::Other)?; + Ok(TestSavedState { value: 0 }) + } + + async fn restore(&mut self, _state: TestSavedState) -> Result<(), RestoreError> { + self.read_guest_memory().map_err(RestoreError::Other) + } + } + + #[derive(MeshPayload)] + struct GuestMemoryDeviceHandle; + + impl ResourceId for GuestMemoryDeviceHandle { + const ID: &'static str = "guest_memory_test"; + } + + struct GuestMemoryDeviceResolver; + + impl ResolveResource + for GuestMemoryDeviceResolver + { + type Output = ResolvedChipsetDevice; + type Error = Infallible; + + fn resolve( + &self, + _resource: GuestMemoryDeviceHandle, + input: ResolveChipsetDeviceHandleParams<'_>, + ) -> Result { + Ok(GuestMemoryDevice { + guest_memory: input.guest_memory.clone(), + encrypted_guest_memory: input.encrypted_guest_memory.clone(), + } + .into()) + } + } + + #[derive(Clone, MeshPayload)] + struct GuestMemoryResolvers; + + impl RemoteDynamicResolvers for GuestMemoryResolvers { + const WORKER_ID_STR: &str = "test_guest_memory_remote_chipset_worker"; + + async fn register_remote_dynamic_resolvers( + self, + resolver: &mut ResourceResolver, + ) -> anyhow::Result<()> { + resolver.add_resolver(GuestMemoryDeviceResolver); + Ok(()) + } + } + async fn new_proxy(vmtime: &VmTimeKeeper) -> ChipsetDeviceProxy { + new_proxy_with( + vmtime, + TestDeviceHandle.into_resource(), + TestResolvers, + GuestMemory::empty(), + GuestMemory::empty(), + ) + .await + } + + /// Creates a proxy for a [`GuestMemoryDevice`] whose guest memory holds + /// the bytes the device expects to read. + async fn new_guest_memory_proxy(vmtime: &VmTimeKeeper) -> ChipsetDeviceProxy { + let guest_memory = GuestMemory::allocate(4096); + guest_memory.write_at(0, &[0x12]).unwrap(); + let encrypted_guest_memory = GuestMemory::allocate(4096); + encrypted_guest_memory.write_at(0, &[0x34]).unwrap(); + new_proxy_with( + vmtime, + GuestMemoryDeviceHandle.into_resource(), + GuestMemoryResolvers, + guest_memory, + encrypted_guest_memory, + ) + .await + } + + async fn new_proxy_with( + vmtime: &VmTimeKeeper, + device: Resource, + dyn_resolvers: T, + guest_memory: GuestMemory, + encrypted_guest_memory: GuestMemory, + ) -> ChipsetDeviceProxy { let (req_send, req_recv) = mesh::channel(); let (resp_send, resp_recv) = mesh::channel(); let (cap_send, cap_recv) = mesh::oneshot(); - let (gm_proxy, gm_remote) = GuestMemoryProxy::new(GuestMemory::empty()); - let (enc_gm_proxy, enc_gm_remote) = GuestMemoryProxy::new(GuestMemory::empty()); - let worker = mesh_worker::launch_local_worker::>( + let (gm_proxy, gm_remote) = GuestMemoryProxy::new(guest_memory); + let (enc_gm_proxy, enc_gm_remote) = GuestMemoryProxy::new(encrypted_guest_memory); + let worker = mesh_worker::launch_local_worker::>( RemoteChipsetDeviceWorkerParameters { - device: TestDeviceHandle.into_resource(), - dyn_resolvers: TestResolvers, + device, + dyn_resolvers, inputs: RemoteChipsetDeviceHandleParams { device_name: "test".to_owned(), is_restoring: false, @@ -523,4 +716,69 @@ mod tests { assert!(proxy.save().await.is_err()); assert!(proxy.restore(state(0)).await.is_err()); } + + #[test] + fn start_services_guest_memory_requests() { + let gm = GuestMemory::allocate(4096); + gm.write_at(0, &[0x12]).unwrap(); + let encrypted_gm = GuestMemory::allocate(4096); + encrypted_gm.write_at(0, &[0x34]).unwrap(); + + let (mut gm_proxy, gm_remote) = GuestMemoryProxy::new(gm); + let (mut enc_gm_proxy, enc_gm_remote) = GuestMemoryProxy::new(encrypted_gm); + let gm_remote = gm_remote.build("remote_gm"); + let enc_gm_remote = enc_gm_remote.build("remote_enc_gm"); + let (req_send, mut req_recv) = mesh::channel(); + + let worker = std::thread::spawn(move || { + let request = pal_async::local::block_on(req_recv.recv()).unwrap(); + let DeviceRequest::Start(rpc) = request else { + panic!("unexpected request"); + }; + rpc.handle_failable_sync(|()| -> anyhow::Result<()> { + let mut data = [0]; + gm_remote.read_at(0, &mut data)?; + assert_eq!(data, [0x12]); + enc_gm_remote.read_at(0, &mut data)?; + assert_eq!(data, [0x34]); + Ok(()) + }); + }); + + pal_async::local::block_on(start_remote_device( + req_send, + &mut gm_proxy, + &mut enc_gm_proxy, + )) + .unwrap(); + worker.join().unwrap(); + } + + #[async_test] + async fn failed_start_cleanup_services_guest_memory_requests(driver: DefaultDriver) { + let vmtime = VmTimeKeeper::new(&driver, VmTime::from_100ns(0)); + let mut proxy = new_guest_memory_proxy(&vmtime).await; + + let error = proxy.start_fallible().await.unwrap_err(); + assert!(format!("{error:#}").contains("start failed")); + // Rolling back the failed start reads guest memory in the remote + // device, which blocks until the parent services the request. + proxy.stop().await; + + proxy.worker.stop(); + proxy.worker.join().await.unwrap(); + } + + #[async_test] + async fn reset_save_restore_service_guest_memory_requests(driver: DefaultDriver) { + let vmtime = VmTimeKeeper::new(&driver, VmTime::from_100ns(0)); + let mut proxy = new_guest_memory_proxy(&vmtime).await; + + proxy.reset().await; + let saved = proxy.save().await.unwrap(); + proxy.restore(saved).await.unwrap(); + + proxy.worker.stop(); + proxy.worker.join().await.unwrap(); + } } diff --git a/workers/chipset_device_worker/src/worker.rs b/workers/chipset_device_worker/src/worker.rs index 6b93bb28258..74c439f816b 100644 --- a/workers/chipset_device_worker/src/worker.rs +++ b/workers/chipset_device_worker/src/worker.rs @@ -252,7 +252,10 @@ impl Worker for RemoteChipsetDeviceWorker { } }, WorkerEvent::DeviceRequest(req) => match req { - DeviceRequest::Start => self.device.start(), + DeviceRequest::Start(rpc) => { + rpc.handle_failable(async |()| self.device.start_fallible().await) + .await + } DeviceRequest::Stop(rpc) => { rpc.handle(async |()| self.device.stop().await).await }