diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index e91a38fb8a4..4b52cdfbf33 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -80,7 +80,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc; #[derive(MeshPayload)] pub enum UhVmRpc { Pause(Rpc<(), bool>), - Resume(Rpc<(), bool>), + Resume(FailableRpc<(), bool>), Save(FailableRpc<(), Vec>), ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume PacketCapture(FailableRpc, PacketCaptureParams>), @@ -226,6 +226,19 @@ pub struct LoadedVmState { pub control_send: mesh::Sender, } +/// The outcome of a worker restart request. +#[expect(clippy::large_enum_variant)] +enum RestartOutcome { + /// The VM state was saved, so the worker can restart. + Restart(FailableRpc<(), T>, ServicingState), + /// The restart failed, but the VM is back in its previous state, so the + /// worker can keep serving. + Recovered, + /// The restart failed and the VM could not be resumed, so the worker + /// cannot serve any more. + Unrecoverable, +} + impl LoadedVm { /// Start running the VM which will start running VTL0. pub async fn run( @@ -235,9 +248,12 @@ impl LoadedVm { correlation_id: Option, mut vm_rpc: mesh::Receiver, mut worker_rpc: mesh::Receiver>, - ) -> Option> { + ) -> anyhow::Result>> { if autostart_vps { - self.start(correlation_id).await; + if let Err(error) = self.start(correlation_id).await { + self.teardown().await; + return Err(error.context("failed to autostart VM")); + } } // VTL2 settings services @@ -325,7 +341,7 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - let state = async { + let outcome = async { let running = self.stop().await; match self .save( @@ -336,26 +352,40 @@ impl LoadedVm { ) .await { - Ok(servicing_state) => Some((rpc, servicing_state)), + Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state), Err(err) => { if running { - self.start(None).await; + if let Err(start_error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + start_error.context(format!( + "worker restart failed ({err:#}) and the VM could not resume" + )), + ))); + return RestartOutcome::Unrecoverable; + } } rpc.complete(Err(RemoteError::new(err))); - None + RestartOutcome::Recovered } } } .instrument(tracing::info_span!("restart", CVM_ALLOWED)) .await; - if let Some((rpc, servicing_state)) = state { - break Some(LoadedVmState { - restart_rpc: rpc, - servicing_state, - vm_rpc, - control_send: self.control_send.lock().take().unwrap(), - }); + match outcome { + RestartOutcome::Restart(rpc, servicing_state) => { + break Some(LoadedVmState { + restart_rpc: rpc, + servicing_state, + vm_rpc, + control_send: self.control_send.lock().take().unwrap(), + }); + } + RestartOutcome::Recovered => {} + // The error has already completed the RPC. Tear + // down through the common path rather than keep a + // worker whose state units can no longer run. + RestartOutcome::Unrecoverable => break None, } } WorkerRpc::Inspect(deferred) => deferred.respond(|resp| { @@ -406,34 +436,39 @@ impl LoadedVm { } Event::UhVmRpc(msg) => match msg { UhVmRpc::Resume(rpc) => { - rpc.handle(async |()| { - if !self.state_units.is_running() { - self.start(None).await; - true - } else { - false - } - }) - .await + if self.state_units.is_running() { + rpc.complete(Ok(false)); + } else if let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM"), + ))); + // A failed start may leave state units terminal; + // this worker cannot safely continue serving. + break None; + } else { + rpc.complete(Ok(true)); + } } UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await, UhVmRpc::Save(rpc) => { - rpc.handle_failable(async |()| { - let running = self.stop().await; - let r = self - .save( - SaveReason::Diagnostic, - None, - KeepAliveConfig::Disabled, - KeepAliveConfig::Disabled, - ) - .await; - if running { - self.start(None).await; - } - r.map(mesh::payload::encode) - }) - .await + let running = self.stop().await; + let result = self + .save( + SaveReason::Diagnostic, + None, + KeepAliveConfig::Disabled, + KeepAliveConfig::Disabled, + ) + .await; + if running && let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM after save"), + ))); + // The state units rolled back and cannot run + // again, so tear down rather than keep this worker. + break None; + } + rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new)); } UhVmRpc::ClearHalt(rpc) => { rpc.handle(async |()| self.partition_unit.clear_halt().await) @@ -585,6 +620,12 @@ impl LoadedVm { } }; + self.teardown().await; + + Ok(state) + } + + async fn teardown(mut self) { // Drain resealing before tearing down any storage transport. Removing // the unit closes its request stream; in-flight I/O finishes first. if let Some(resealer) = self.hardware_reseal.take() { @@ -602,8 +643,6 @@ impl LoadedVm { if let Some(vmbus) = self.vmbus_server { vmbus.remove().await.shutdown().await; } - - state } /// Handles a servicing request from the host. @@ -654,7 +693,9 @@ impl LoadedVm { .context("failed to notify host of servicing-while-paused failure")?; if running { - self.start(Some(correlation_id)).await; + self.start(Some(correlation_id)) + .await + .context("failed to resume VM after servicing failure")?; } false } @@ -884,8 +925,8 @@ impl LoadedVm { } } - async fn start(&mut self, correlation_id: Option) { - self.state_units.start().await; + async fn start(&mut self, correlation_id: Option) -> anyhow::Result<()> { + self.state_units.start().await?; // Log the boot/blackout time. let reference_time = ReferenceTime::new(self.partition.reference_time()); @@ -911,6 +952,8 @@ impl LoadedVm { "starting VM" ) } + + Ok(()) } /// Returns true if the VM was previously running. @@ -1137,7 +1180,20 @@ impl LoadedVm { ) .await?; - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + if let Err(remove_error) = self + .network_settings + .as_mut() + .unwrap() + .remove_network(instance_id) + .await + { + return Err(error.context(format!( + "failed to remove network after start failure: {remove_error:#}" + ))); + } + return Err(error); + } self.emuplat_servicing.netvsp_state.push(save_state); Ok(()) diff --git a/openhcl/underhill_core/src/lib.rs b/openhcl/underhill_core/src/lib.rs index 329d1c119f8..69cb928ad5c 100644 --- a/openhcl/underhill_core/src/lib.rs +++ b/openhcl/underhill_core/src/lib.rs @@ -686,11 +686,18 @@ async fn run_control( continue; }; - let was_resumed = workers + let was_resumed = match workers .vm_rpc .call(UhVmRpc::Resume, ()) .await - .context("failed to resumed VM")?; + .context("failed to resume VM")? + { + Ok(was_resumed) => was_resumed, + Err(error) => { + rpc.complete(Err(error)); + continue; + } + }; let was_halted = workers .vm_rpc diff --git a/openhcl/underhill_core/src/worker.rs b/openhcl/underhill_core/src/worker.rs index 1039d11cfaa..df433f73e24 100644 --- a/openhcl/underhill_core/src/worker.rs +++ b/openhcl/underhill_core/src/worker.rs @@ -468,6 +468,7 @@ impl Worker for UnderhillVmWorker { )); tracing::info!(CVM_ALLOWED, "terminating worker"); self.get_thread.join().unwrap(); + let state = state?; if let Some(state) = state { let params = UnderhillWorkerParameters { env_cfg: self.env_cfg, diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 6566a94a8ec..e2297dec9a8 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3682,7 +3682,12 @@ impl LoadedVm { if self.running { return false; } - self.state_units.start().await; + // Resume cannot report a failure to its callers, so treat a unit that + // fails to start like any other unrecoverable state unit failure. + self.state_units + .start() + .await + .expect("state units failed to start"); self.running = true; true } @@ -3724,8 +3729,10 @@ impl LoadedVm { // Start state units so device config space is accessible. This also // freezes SMMU capabilities before firmware reads them. - self.state_units.start().await; - + self.state_units + .start() + .await + .context("failed to start devices for PCI resource assignment")?; let result = ecam_config_access::assign_pci_resources_for_root_complexes( &self.inner.chipset, &self.inner.pcie_host_bridges, @@ -3832,7 +3839,12 @@ impl LoadedVm { } Err(err) => { if stopped { - self.state_units.start().await; + if let Err(start_error) = self.state_units.start().await { + rpc.complete(Err(RemoteError::new(start_error.context( + "worker restart failed and the VM could not resume", + )))); + break; + } } rpc.complete(Err(RemoteError::new(err))); } @@ -3852,7 +3864,16 @@ impl LoadedVm { Event::VmRpc(Err(_)) => break, Event::VmRpc(Ok(message)) => match message { VmRpc::Reset(rpc) => { - rpc.handle_failable(async |()| self.reset(true).await).await + let result = self.reset(true).await; + // The reset pauses the VM, so state units that still + // run after it fails were left by a failed start, and + // can never start again. Tear the VM down, as after a + // failed restart. + let start_failed = result.is_err() && self.state_units.is_running(); + rpc.complete(result.map_err(RemoteError::new)); + if start_failed { + break; + } } VmRpc::ClearHalt(rpc) => { rpc.handle(async |()| self.inner.partition_unit.clear_halt().await) @@ -3905,8 +3926,12 @@ impl LoadedVm { resource, ) .await?; + if let Err(error) = self.state_units.start_stopped_units().await { + // Dropping the channel unit revokes the channel. + drop(device.remove().await); + return Err(error); + } self.inner.vmbus_devices.push(device); - self.state_units.start_stopped_units().await; anyhow::Ok(()) }) .await @@ -4072,7 +4097,15 @@ impl LoadedVm { // MSI. The guest may begin probing config space // immediately after receiving the interrupt, so // the device must be ready first. - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let (_, unit, _) = self + .inner + .pcie_hotplug_devices + .pop() + .expect("device was just pushed"); + unit.remove().await; + return Err(error); + } // Now attach the device and notify the guest. if let Err(e) = rc.lock().hotplug_add_device( @@ -4243,7 +4276,13 @@ impl LoadedVm { } device_removal.drain().await; - self.pause().await; + // Stop the state units whenever they run, not only when the VM does: a + // failed start, as in a reset, can leave units running while the VM is + // paused, and a unit whose start is uncertain may not answer its stop. + if self.state_units.is_running() { + self.state_units.stop_for_teardown().await; + } + self.running = false; self.teardown_chipset_devices().await; self.inner.partition_unit.teardown().await; if let Some(vmbus) = self.inner.vmbus_server { diff --git a/vm/chipset_device_resources/src/lib.rs b/vm/chipset_device_resources/src/lib.rs index 9d2bab63d58..8abd0872803 100644 --- a/vm/chipset_device_resources/src/lib.rs +++ b/vm/chipset_device_resources/src/lib.rs @@ -13,6 +13,7 @@ use std::ops::RangeInclusive; use vm_resource::CanResolveTo; use vm_resource::kind::ChipsetDeviceHandleKind; use vmcore::device_state::ChangeDeviceState; +use vmcore::device_state::InputHookCompletion; use vmcore::line_interrupt::LineInterrupt; use vmcore::save_restore::AsyncProtobufSaveRestore; use vmcore::vm_task::VmTaskDriverSource; @@ -92,6 +93,8 @@ pub trait ConfigureChipsetDevice: Send { #[async_trait] trait DynChipsetDevice: ChipsetDevice + AsyncProtobufSaveRestore + InspectMut { fn start(&mut self); + fn quiesce_input(&mut self) -> InputHookCompletion; + fn resume_input(&mut self) -> InputHookCompletion; async fn stop(&mut self); async fn reset(&mut self); async fn teardown(self: Box); @@ -104,6 +107,12 @@ impl InputHookCompletion { + self.quiesce_input() + } + fn resume_input(&mut self) -> InputHookCompletion { + self.resume_input() + } async fn stop(&mut self) { self.stop().await } @@ -136,6 +145,14 @@ impl ChangeDeviceState for ErasedChipsetDevice { self.0.start() } + fn quiesce_input(&mut self) -> InputHookCompletion { + self.0.quiesce_input() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.0.resume_input() + } + async fn stop(&mut self) { self.0.stop().await } diff --git a/vm/devices/pci/vpci_relay/src/lib.rs b/vm/devices/pci/vpci_relay/src/lib.rs index 22e971f1f90..76bae759c18 100644 --- a/vm/devices/pci/vpci_relay/src/lib.rs +++ b/vm/devices/pci/vpci_relay/src/lib.rs @@ -490,7 +490,7 @@ impl VpciRelay { .await? }; - entry.insert(RelayedDevice { + let device = RelayedDevice { bus_instance_id: instance_id, bus_client: vpci_client, vpci_device: vpci_device.clone(), @@ -498,9 +498,13 @@ impl VpciRelay { bus_unit, device_unit, ready_to_remove: false, - }); + }; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + device.remove().await; + return Err(error); + } + entry.insert(device); Ok(()) } } diff --git a/vm/vmcore/src/device_state.rs b/vm/vmcore/src/device_state.rs index 4dcdcb24156..92bf2ea5941 100644 --- a/vm/vmcore/src/device_state.rs +++ b/vm/vmcore/src/device_state.rs @@ -6,6 +6,41 @@ #![forbid(unsafe_code)] use std::future::Future; +use std::pin::Pin; +use std::task::Context; +use std::task::Poll; + +/// The completion of a device's input hook, such as +/// [`ChangeDeviceState::quiesce_input`]. +/// +/// An input hook runs synchronously, with exclusive access to the device. The +/// work that it must wait for, such as a request to a device task, completes in +/// this future instead, which the caller awaits after it gives up that access. +#[must_use = "an input hook is complete only once its completion is awaited"] +pub struct InputHookCompletion(Option> + Send>>>); + +impl InputHookCompletion { + /// Returns the completion of a hook that has nothing left to wait for. + pub fn ready() -> Self { + Self(None) + } + + /// Returns a completion that waits for `future`. + pub fn new(future: impl 'static + Send + Future>) -> Self { + Self(Some(Box::pin(future))) + } +} + +impl Future for InputHookCompletion { + type Output = anyhow::Result<()>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match &mut self.get_mut().0 { + Some(future) => future.as_mut().poll(cx), + None => Poll::Ready(Ok(())), + } + } +} /// Trait for transitioning device state. pub trait ChangeDeviceState { @@ -27,6 +62,31 @@ pub trait ChangeDeviceState { // to finish, which may require a bunch of other changes. fn start(&mut self); + /// Stops accepting new host input before a snapshot vCPU boundary. + /// + /// This runs with exclusive access to the device, which for a chipset + /// device means with its lock held while vCPUs may still access the device, + /// so it must not block. The work that stopping input must wait for, such + /// as a request to a device task, belongs in the returned completion, which + /// the caller awaits after it releases the device. Meanwhile, the device + /// keeps serving vCPUs and being polled. + /// + /// [`ChangeDeviceState::resume_input`] may be called even if the completion + /// fails or is not awaited. + fn quiesce_input(&mut self) -> InputHookCompletion { + InputHookCompletion::ready() + } + + /// Resumes host input after a failed snapshot transaction. + /// + /// This must be safe whether or not [`ChangeDeviceState::quiesce_input`] + /// completed and idempotent across repeated recovery attempts. Like that + /// hook, it must not block, and the caller awaits its completion after it + /// releases the device. + fn resume_input(&mut self) -> InputHookCompletion { + InputHookCompletion::ready() + } + /// Pauses guest-request processing in a save-consistent state. /// /// After this returns, the device must not process additional guest requests diff --git a/vmm_core/src/device_builder.rs b/vmm_core/src/device_builder.rs index 089ac1444c9..440d98a9dc4 100644 --- a/vmm_core/src/device_builder.rs +++ b/vmm_core/src/device_builder.rs @@ -144,7 +144,11 @@ pub async fn build_dynamic_vpci_device( }; let pending_offer = pending_offer.context("missing deferred VPCI channel offer")?; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + vpci_unit.remove().await; + pci_unit.remove().await; + return Err(error); + } if let Err(error) = pending_offer .offer_registered(&vpci_bus, driver_source, vmbus, state_units.is_running()) .await diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 87a9525e73d..8a8192967c1 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -30,6 +30,9 @@ #![forbid(unsafe_code)] +mod quiesce; +mod start; + use futures::FutureExt; use futures::StreamExt; use futures::future::join_all; @@ -39,6 +42,7 @@ use inspect::InspectMut; use mesh::MeshPayload; use mesh::Receiver; use mesh::Sender; +use mesh::error::RemoteError; use mesh::payload::Protobuf; use mesh::rpc::FailableRpc; use mesh::rpc::Rpc; @@ -62,6 +66,7 @@ use std::sync::atomic::Ordering; use std::time::Instant; use thiserror::Error; use tracing::Instrument; +use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; use vmcore::save_restore::SavedStateBlob; @@ -70,11 +75,28 @@ use vmcore::save_restore::SavedStateBlob; #[derive(Debug, MeshPayload)] pub enum StateRequest { /// Start asynchronous operations. + /// + /// Once this request is delivered, a [`StateRequest::Stop`] may follow + /// even if its response is not observed. Start(Rpc<(), ()>), /// Stop asynchronous operations. + /// + /// The handler must be safe after any delivered start attempt. Stop(Rpc<(), ()>), + /// Stop accepting new host input before a snapshot vCPU boundary. + /// + /// A [`StateRequest::ResumeInput`] may follow even if this request returns + /// an error or its response is not observed. + QuiesceInput(FailableRpc<(), ()>), + + /// Resume host input after a failed snapshot transaction. + /// + /// The handler must be safe whether or not input quiesce completed and + /// idempotent across repeated recovery attempts. + ResumeInput(FailableRpc<(), ()>), + /// Reset a stopped unit to initial state. Reset(FailableRpc<(), ()>), @@ -101,6 +123,26 @@ pub trait StateUnit: InspectMut { /// Stop asynchronous processing. async fn stop(&mut self); + /// Stops accepting new host input while preserving runtime state. + /// + /// This runs synchronously, and the work that it must wait for completes + /// in the returned completion, which is awaited before the request is + /// answered. [`StateUnit::resume_input`] may be called even if the + /// completion fails or is not awaited. + fn quiesce_input(&mut self) -> InputHookCompletion { + InputHookCompletion::ready() + } + + /// Resumes host input after a failed snapshot transaction. + /// + /// This must be safe whether or not [`StateUnit::quiesce_input`] completed + /// and idempotent across repeated recovery attempts. Like that hook, it + /// runs synchronously, and its completion is awaited before the request is + /// answered. + fn resume_input(&mut self) -> InputHookCompletion { + InputHookCompletion::ready() + } + /// Reset to initial state. /// /// Must only be called while stopped. @@ -166,6 +208,8 @@ impl StateRequest { StateRequest::Start(_) | StateRequest::Stop(_) + | StateRequest::QuiesceInput(_) + | StateRequest::ResumeInput(_) | StateRequest::Reset(_) | StateRequest::Save(_) | StateRequest::Restore(_) => { @@ -194,6 +238,8 @@ impl StateRequest { match self { StateRequest::Start(rpc) => rpc.handle(async |()| unit.start().await).await, StateRequest::Stop(rpc) => rpc.handle(async |()| unit.stop().await).await, + StateRequest::QuiesceInput(rpc) => complete_input_hook(rpc, unit.quiesce_input()).await, + StateRequest::ResumeInput(rpc) => complete_input_hook(rpc, unit.resume_input()).await, StateRequest::Reset(rpc) => rpc.handle_failable(async |()| unit.reset().await).await, StateRequest::Save(rpc) => rpc.handle_failable(async |()| unit.save().await).await, StateRequest::Restore(rpc) => { @@ -205,6 +251,15 @@ impl StateRequest { } } +/// Answers `rpc` once the completion of an input hook finishes. +/// +/// This is not generic, so that the state requests of every unit type share +/// it. +async fn complete_input_hook(rpc: FailableRpc<(), ()>, completion: InputHookCompletion) { + let result = completion.await; + rpc.complete(result.map_err(RemoteError::new)); +} + /// A set of state units. #[derive(Debug)] pub struct StateUnits { @@ -222,6 +277,7 @@ enum State { Resetting, Saving, Restoring, + StartUncertain, } #[derive(Debug)] @@ -486,41 +542,87 @@ impl StateUnits { self.running } + fn check_no_terminal_units(&self, op: &'static str) -> Result<(), StateTransitionError> { + let inner = self.inner.lock(); + check( + op, + inner + .units + .values() + .filter(|unit| matches!(unit.state, State::StartUncertain)) + .map(|unit| { + ( + unit.name.clone(), + Err::<(), _>(anyhow::anyhow!( + "unit is in terminal state {:?}", + unit.state + )), + ) + }), + ) + } + /// Starts any units that are individually stopped, because they were added /// via [`StateUnits::add`] while the VM was running. /// /// Does nothing if all units are stopped, via [`StateUnits::stop`]. - pub async fn start_stopped_units(&mut self) { + pub async fn start_stopped_units(&mut self) -> anyhow::Result<()> { if self.is_running() { - self.start().await; + self.start().await?; } + Ok(()) } /// Starts all the state units. - pub async fn start(&mut self) { + /// + /// If a unit's task does not answer its start request, the start fails, + /// and the units that did start are stopped again. If a unit may still be + /// running after that, [`StateUnits::is_running`] returns true, so that + /// [`StateUnits::stop`] can stop it. + pub async fn start(&mut self) -> anyhow::Result<()> { self.restore_allowed = false; - self.run_op( - "start", - None, - State::Stopped, - State::Starting, - State::Running, - StateRequest::Start, - |_, _| Some(()), - |unit| Some(&unit.dependencies), - ) - .await; - self.running = true; + self.start_with_rollback().await } /// Stops all the state units. + /// + /// This also stops the units whose start is uncertain. pub async fn stop(&mut self) { + self.stop_units(false).await + } + + /// Stops all the state units before the VM is torn down. + /// + /// Unlike [`StateUnits::stop`], this does not panic when a unit whose start + /// is uncertain does not answer its stop, as when its task has exited. It + /// logs the failure and stops the other units, including the units that + /// such a unit depends on, since the VM is not run again. + pub async fn stop_for_teardown(&mut self) { + self.stop_units(true).await + } + + async fn stop_units(&mut self, for_teardown: bool) { assert!(self.running); + // Make units whose start is uncertain receive Stop, then restore their + // terminal markers after the ordered operation. + let uncertain_ids = { + let mut inner = self.inner.lock(); + inner + .units + .iter_mut() + .filter(|(_, unit)| unit.state == State::StartUncertain) + .map(|(&id, unit)| { + unit.state = State::Running; + id + }) + .collect::>() + }; // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( "stop", None, + if for_teardown { &uncertain_ids } else { &[] }, State::Running, State::Stopping, State::Stopped, @@ -529,6 +631,13 @@ impl StateUnits { |unit| Some(&unit.dependents), ) .await; + let mut inner = self.inner.lock(); + for id in uncertain_ids { + if let Some(unit) = inner.units.get_mut(&id) { + assert_eq!(unit.state, State::Stopped); + unit.state = State::StartUncertain; + } + } self.running = false; } @@ -537,6 +646,7 @@ impl StateUnits { /// Panics if running. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("reset")?; self.restore_allowed = false; // Reset in dependency order so that dependants observe their // dependencies' reset state. @@ -544,6 +654,7 @@ impl StateUnits { .run_op( "reset", None, + &[], State::Stopped, State::Resetting, State::Stopped, @@ -563,12 +674,14 @@ impl StateUnits { /// Panics if running. pub async fn save(&mut self) -> Result, StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("save")?; // Save can occur in any order since it will not observably mutate // state. let r = self .run_op( "save", None, + &[], State::Stopped, State::Saving, State::Stopped, @@ -601,6 +714,7 @@ impl StateUnits { states: Vec, ) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("restore")?; if !self.restore_allowed { return Err(StateTransitionError { op: "restore", @@ -644,6 +758,7 @@ impl StateUnits { .run_op( "restore", None, + &[], State::Stopped, State::Restoring, State::Stopped, @@ -675,6 +790,10 @@ impl StateUnits { /// `unit_ids` is the set of the units whose state should be changed. If /// `unit_ids` is `None`, all units change states. /// + /// `tolerated_ids` is the set of the units whose failure to answer the + /// request is logged instead of treated as an unrecoverable unit failure, + /// such as the units whose start is uncertain when the VM is torn down. + /// /// The old state for each unit must be `old_state`. During the operation, /// the unit is temporarily places in `interim_state`. When complete, the /// unit is placed in `new_state`. @@ -693,6 +812,7 @@ impl StateUnits { &self, op: &str, unit_ids: Option<&[u64]>, + tolerated_ids: &[u64], old_state: State, interim_state: State, new_state: State, @@ -751,20 +871,12 @@ impl StateUnits { let mut inner = self.inner.lock(); let r = results .into_iter() - .filter_map(|(name, id, r)| { - match r { - Ok(Some(r)) => Some((name, r)), - Ok(None) => None, - Err(err) => { - // If the unit was removed during the operation, then - // ignore the failure. Otherwise, panic because unit - // failure is not recoverable. FUTURE: reconsider this - // position. - if inner.units.contains_key(&id) { - panic!("{:?}", err); - } - None - } + .filter_map(|(name, id, r)| match r { + Ok(Some(r)) => Some((name, r)), + Ok(None) => None, + Err(err) => { + inner.unanswered_request(id, tolerated_ids, err); + None } }) .collect(); @@ -808,6 +920,22 @@ impl Inner { }; ReadySet(Arc::new(units)) } + + /// Handles a state change request that unit `id` did not answer. + /// + /// If the unit is in `tolerated_ids`, log the failure. If the unit was + /// removed during the operation, ignore it. Otherwise, panic because unit + /// failure is not recoverable. FUTURE: reconsider this position. + fn unanswered_request(&self, id: u64, tolerated_ids: &[u64], err: UnitRecvError) { + if tolerated_ids.contains(&id) { + tracing::error!( + error = &err as &dyn std::error::Error, + "unit whose start is uncertain did not answer" + ); + } else if self.units.contains_key(&id) { + panic!("{:?}", err); + } + } } #[derive(Clone)] @@ -1047,6 +1175,9 @@ impl Ready { #[cfg(test)] mod tests { + mod quiesce; + mod start; + use super::StateTransitionErrorKind; use super::StateUnit; use super::StateUnits; @@ -1219,7 +1350,7 @@ mod tests { let state = units.save().await.unwrap(); units.restore(state).await.unwrap(); assert_eq!(calls.load(Ordering::Relaxed), 2); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); assert!(units.restore(state).await.is_err()); @@ -1293,13 +1424,13 @@ mod tests { .add("b") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) .unwrap(); - units.start().await; + units.start().await.unwrap(); let _c = units .add("c") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)); units.stop().await; - units.start().await; + units.start().await.unwrap(); units.stop().await; @@ -1347,7 +1478,7 @@ mod tests { ) }) .unwrap(); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); @@ -1457,7 +1588,7 @@ mod tests { ); } - units.start().await; + units.start().await.unwrap(); units.stop().await; a.remove().await; @@ -1467,7 +1598,7 @@ mod tests { assert!(inner.units[&c.handle().id.id].dependencies.is_empty()); } - units.start().await; + units.start().await.unwrap(); units.stop().await; // Removing the remaining unit leaves the set empty. diff --git a/vmm_core/state_unit/src/quiesce.rs b/vmm_core/state_unit/src/quiesce.rs new file mode 100644 index 00000000000..fd27ef627f7 --- /dev/null +++ b/vmm_core/state_unit/src/quiesce.rs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Snapshot input gating: stopping host input before the snapshot vCPU +//! boundary, and resuming it after a failed snapshot transaction. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use mesh::rpc::FailableRpc; + +impl StateUnits { + /// Stops host-input producers in running units before establishing a + /// snapshot vCPU boundary. + /// + /// Waits for every running unit, without a deadline, so a unit that does + /// not answer stalls the transaction. Stopped units are skipped, and + /// terminal lifecycle states fail the transaction. Drive the returned + /// future to completion: if it is dropped, the units still run the hooks + /// that it requested, and a unit's next request may arrive before its hook + /// completes. + pub async fn quiesce_input_for_save(&mut self) -> anyhow::Result<()> { + self.run_input_op("quiesce_input", StateRequest::QuiesceInput) + .await + } + + /// Resumes host-input producers in running units before releasing a failed + /// snapshot boundary. + /// + /// Waits for every running unit, without a deadline, so a unit that does + /// not answer stalls the transaction. Stopped units are skipped, and + /// terminal lifecycle states fail the transaction. As for + /// [`StateUnits::quiesce_input_for_save`], drive the returned future to + /// completion. + pub async fn resume_input_after_save(&mut self) -> anyhow::Result<()> { + self.run_input_op("resume_input", StateRequest::ResumeInput) + .await + } + + async fn run_input_op( + &mut self, + operation: &'static str, + request: impl Copy + FnOnce(FailableRpc<(), ()>) -> StateRequest, + ) -> anyhow::Result<()> { + let mut operations = Vec::new(); + let mut failures = Vec::new(); + { + let inner = self.inner.lock(); + for unit in inner.units.values() { + match unit.state { + State::Running => { + let name = unit.name.clone(); + let operation = state_change(name.clone(), unit, request, Some(())); + operations.push(async move { (name, operation.await) }); + } + State::Stopped => {} + State::StartUncertain => { + failures.push(format!( + "{}: unit is in terminal state {:?}", + unit.name, unit.state + )); + } + state => { + failures.push(format!( + "{}: unit is in unexpected state {state:?}", + unit.name + )); + } + } + } + } + + for (name, result) in join_all(operations).await { + match result { + Ok(Some(Ok(()))) => {} + Ok(Some(Err(error))) => failures.push(format!("{name}: {error}")), + Ok(None) => failures.push(format!("{name}: request was not sent")), + Err(error) => failures.push(format!("{name}: {error}")), + } + } + + anyhow::ensure!( + failures.is_empty(), + "{operation} failed: {}", + failures.join("; ") + ); + Ok(()) + } +} diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs new file mode 100644 index 00000000000..896a6fa0559 --- /dev/null +++ b/vmm_core/state_unit/src/start.rs @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Start with rollback. A unit's start cannot fail, but its task can stop +//! answering: then its dependents do not start, and the units that did start +//! are stopped again, rather than the whole process panicking. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use std::future::Future; +use std::sync::Arc; +use std::time::Instant; + +/// The outcome of a unit's start. +enum StartResult { + Started, + DependencyFailed, + Uncertain(anyhow::Error), +} + +/// The outcome of a unit's start, with the unit's name and ID. +type UnitStartResult = (Arc, u64, StartResult); + +impl StateUnits { + /// Starts all stopped units in dependency order. + /// + /// Fails without starting any unit if one is in a terminal state. A unit + /// does not start if one of its dependencies did not start. If a unit's + /// task does not answer its start request, the units that did start are + /// stopped again, and the unit is left in a terminal state that fails later + /// starts. A started unit that cannot be stopped again, because a unit that + /// depends on it was not stopped, is left running in that terminal state + /// too. Because a unit in that state may be running, the units then stay + /// running, so that [`StateUnits::stop`] can stop it. + pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { + let was_running = self.running; + let operations = self.start_operations()?; + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "state-unit start complete"); + + let (started_ids, mut failures) = self.record_start_results(results); + if failures.is_empty() { + self.running = true; + Ok(()) + } else { + failures.extend(self.stop_after_failed_start(&started_ids).await); + self.running = was_running + || self + .inner + .lock() + .units + .values() + .any(|unit| unit.state != State::Stopped); + anyhow::bail!("state units could not start: {}", failures.join("; ")) + } + } + + /// Marks the stopped units as starting, and returns the operations that + /// start them in dependency order. + /// + /// Fails without changing any unit if one is in a terminal state. + fn start_operations( + &self, + ) -> anyhow::Result + use<>>> { + let mut inner = self.inner.lock(); + if let Some(unit) = inner + .units + .values() + .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) + { + anyhow::bail!( + "state unit '{}' cannot be started from terminal state {:?}", + unit.name, + unit.state + ); + } + let ready_set = inner.ready_set(None); + let mut operations = Vec::new(); + for (&id, unit) in &mut inner.units { + if unit.state == State::Running { + ready_set.done(id, true); + continue; + } + assert_eq!( + unit.state, + State::Stopped, + "unit {} is not stopped before start", + unit.name, + ); + let name = unit.name.clone(); + let dependencies = unit.dependencies.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + operations.push(async move { + if !ready_set.wait("start", id, &dependencies).await { + ready_set.done(id, false); + return (name, id, StartResult::DependencyFailed); + } + + let result = match start.await { + Ok(Some(())) => StartResult::Started, + Ok(None) => { + StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) + } + Err(error) => StartResult::Uncertain(error.into()), + }; + ready_set.done(id, matches!(result, StartResult::Started)); + (name, id, result) + }); + unit.state = State::Starting; + } + Ok(operations) + } + + /// Moves each unit to the state that the outcome of its start leaves it in, + /// and returns the IDs of the units that started and the failures. + fn record_start_results(&self, results: Vec) -> (Vec, Vec) { + let mut started_ids = Vec::new(); + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + match result { + StartResult::Started => { + unit.state = State::Running; + started_ids.push(id); + } + StartResult::DependencyFailed => { + unit.state = State::Stopped; + failures.push(format!("{name}: a dependency did not start")); + } + StartResult::Uncertain(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {error:#}")); + } + } + } + (started_ids, failures) + } + + /// Stops the units in `started_ids` again in reverse dependency order. + /// + /// Every other unit that is not stopped, such as one that was already + /// running or one whose start is uncertain, blocks the stop of its + /// dependencies without receiving a stop request. A unit whose stop is + /// blocked or not confirmed is left in a terminal state. + async fn stop_after_failed_start(&self, started_ids: &[u64]) -> Vec { + const OP: &str = "failed_start_rollback"; + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + ready_set = inner.ready_set(None); + for (&id, unit) in &inner.units { + if !started_ids.contains(&id) { + ready_set.done(id, unit.state == State::Stopped); + } + } + for (&id, unit) in inner + .units + .iter_mut() + .filter(|(id, _)| started_ids.contains(id)) + { + assert_eq!(unit.state, State::Running); + let name = unit.name.clone(); + let dependents = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); + operations.push(async move { + if !ready_set.wait(OP, id, &dependents).await { + ready_set.done(id, false); + return (name, id, None); + } + let result = stop.await; + ready_set.done(id, matches!(&result, Ok(Some(())))); + (name, id, Some(result)) + }); + unit.state = State::Stopping; + } + } + + let results = join_all(operations).await; + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + match result { + Some(Ok(Some(()))) => unit.state = State::Stopped, + Some(Ok(None)) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {OP} stop request was not sent")); + } + Some(Err(error)) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {OP} stop failed: {error:#}")); + } + None => { + unit.state = State::StartUncertain; + failures.push(format!( + "{name}: {OP} stop skipped because a dependent was not stopped" + )); + } + } + } + failures + } +} diff --git a/vmm_core/state_unit/src/tests/quiesce.rs b/vmm_core/state_unit/src/tests/quiesce.rs new file mode 100644 index 00000000000..8ca9fa62a3c --- /dev/null +++ b/vmm_core/state_unit/src/tests/quiesce.rs @@ -0,0 +1,286 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for gating host input around the snapshot boundary. + +use crate::StateRequest; +use crate::StateUnit; +use crate::StateUnits; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use mesh::Receiver; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::time::Duration; +use std::time::Instant; +use test_with_tracing::test; +use vmcore::device_state::InputHookCompletion; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +#[derive(Default)] +struct InputCounts { + quiesces: AtomicUsize, + resumes: AtomicUsize, +} + +struct InputTestUnit { + driver: DefaultDriver, + counts: Arc, + delay: Option, + fail_quiesce: bool, + fail_resume: bool, +} + +impl StateUnit for InputTestUnit { + async fn start(&mut self) {} + + async fn stop(&mut self) {} + + fn quiesce_input(&mut self) -> InputHookCompletion { + self.counts.quiesces.fetch_add(1, Ordering::Relaxed); + let fail = self.fail_quiesce; + let delay = self.delay.map(|delay| (self.driver.clone(), delay)); + InputHookCompletion::new(async move { + if let Some((driver, delay)) = delay { + pal_async::timer::PolledTimer::new(&driver) + .sleep(delay) + .await; + } + anyhow::ensure!(!fail, "intentional input quiesce failure"); + Ok(()) + }) + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.counts.resumes.fetch_add(1, Ordering::Relaxed); + let fail = self.fail_resume; + InputHookCompletion::new(async move { + anyhow::ensure!(!fail, "intentional input resume failure"); + Ok(()) + }) + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for InputTestUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +fn input_unit(driver: &DefaultDriver, counts: Arc) -> InputTestUnit { + InputTestUnit { + driver: driver.clone(), + counts, + delay: None, + fail_quiesce: false, + fail_resume: false, + } +} + +/// Runs `unit` like [`run_unit`], but after answering `answered_starts` start +/// requests, starts the unit for each later one without answering it, so that +/// the start is uncertain while the unit still answers other requests. +async fn run_unit_leaving_starts_unanswered( + mut unit: T, + mut recv: Receiver, + answered_starts: usize, +) -> T { + let mut starts = 0; + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) if starts >= answered_starts => { + unit.start().await; + drop(response); + } + request => { + if matches!(request, StateRequest::Start(_)) { + starts += 1; + } + request.apply(&mut unit).await; + } + } + } + unit +} + +#[async_test] +async fn test_input_hooks_run_only_for_running_units(driver: DefaultDriver) { + let units = StateUnits::new(); + let running_counts = Arc::new(InputCounts::default()); + let _running = units + .add("running") + .spawn(&driver, |recv| { + run_unit(input_unit(&driver, running_counts.clone()), recv) + }) + .unwrap(); + let mut units = units; + units.start().await.unwrap(); + + let stopped_counts = Arc::new(InputCounts::default()); + let _stopped = units + .add("stopped") + .spawn(&driver, |recv| { + run_unit(input_unit(&driver, stopped_counts.clone()), recv) + }) + .unwrap(); + + units.quiesce_input_for_save().await.unwrap(); + units.resume_input_after_save().await.unwrap(); + + assert_eq!(running_counts.quiesces.load(Ordering::Relaxed), 1); + assert_eq!(running_counts.resumes.load(Ordering::Relaxed), 1); + assert_eq!(stopped_counts.quiesces.load(Ordering::Relaxed), 0); + assert_eq!(stopped_counts.resumes.load(Ordering::Relaxed), 0); +} + +#[async_test] +async fn test_input_hook_errors_are_reported_per_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let successful_counts = Arc::new(InputCounts::default()); + let _successful = units + .add("successful") + .spawn(&driver, |recv| { + run_unit(input_unit(&driver, successful_counts.clone()), recv) + }) + .unwrap(); + let failing_counts = Arc::new(InputCounts::default()); + let _failing = units + .add("failing") + .spawn(&driver, |recv| { + let mut unit = input_unit(&driver, failing_counts.clone()); + unit.fail_quiesce = true; + unit.fail_resume = true; + run_unit(unit, recv) + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units.quiesce_input_for_save().await.unwrap_err(); + assert!( + error + .to_string() + .contains("failing: intentional input quiesce failure") + ); + let error = units.resume_input_after_save().await.unwrap_err(); + assert!( + error + .to_string() + .contains("failing: intentional input resume failure") + ); + assert_eq!(successful_counts.quiesces.load(Ordering::Relaxed), 1); + assert_eq!(successful_counts.resumes.load(Ordering::Relaxed), 1); + assert_eq!(failing_counts.quiesces.load(Ordering::Relaxed), 1); + assert_eq!(failing_counts.resumes.load(Ordering::Relaxed), 1); +} + +#[async_test] +async fn test_input_hook_closed_channel_is_reported(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _closed = units + .add("closed") + .spawn(&driver, |mut recv| async move { + let request = recv.next().await.unwrap(); + let StateRequest::Start(response) = request else { + panic!("unexpected request"); + }; + response.complete(()); + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units.quiesce_input_for_save().await.unwrap_err(); + assert!(error.to_string().contains("quiesce_input failed: closed:")); +} + +#[async_test] +async fn test_slow_input_hook_is_awaited(driver: DefaultDriver) { + let delay = Duration::from_millis(200); + let mut units = StateUnits::new(); + let failing_counts = Arc::new(InputCounts::default()); + let _failing = units + .add("failing") + .spawn(&driver, |recv| { + let mut unit = input_unit(&driver, failing_counts.clone()); + unit.fail_quiesce = true; + run_unit(unit, recv) + }) + .unwrap(); + let counts = Arc::new(InputCounts::default()); + let _slow = units + .add("slow") + .spawn(&driver, |recv| { + let mut unit = input_unit(&driver, counts.clone()); + unit.delay = Some(delay); + run_unit(unit, recv) + }) + .unwrap(); + units.start().await.unwrap(); + + // The transaction has no deadline: it waits for the slow hook, which + // succeeds, and reports only the hook that failed. + let started = Instant::now(); + let error = units.quiesce_input_for_save().await.unwrap_err(); + assert!(started.elapsed() >= delay, "{:?}", started.elapsed()); + let error = error.to_string(); + assert!(error.contains("failing: intentional input quiesce failure")); + assert!(!error.contains("slow"), "{error}"); + assert_eq!(failing_counts.quiesces.load(Ordering::Relaxed), 1); + assert_eq!(counts.quiesces.load(Ordering::Relaxed), 1); + + // Resuming input reaches both units, and a stop follows. + units.resume_input_after_save().await.unwrap(); + assert_eq!(failing_counts.resumes.load(Ordering::Relaxed), 1); + assert_eq!(counts.resumes.load(Ordering::Relaxed), 1); + units.stop().await; +} + +#[async_test] +async fn test_input_hooks_reject_terminal_units(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _running = units + .add("running") + .spawn(&driver, |recv| { + run_unit(input_unit(&driver, Arc::new(InputCounts::default())), recv) + }) + .unwrap(); + units.start().await.unwrap(); + + let terminal_counts = Arc::new(InputCounts::default()); + let _terminal = units + .add("terminal") + .spawn(&driver, |recv| { + run_unit_leaving_starts_unanswered( + input_unit(&driver, terminal_counts.clone()), + recv, + 0, + ) + }) + .unwrap(); + units.start_stopped_units().await.unwrap_err(); + + let error = units.quiesce_input_for_save().await.unwrap_err(); + assert!( + error + .to_string() + .contains("terminal: unit is in terminal state StartUncertain") + ); + assert_eq!(terminal_counts.quiesces.load(Ordering::Relaxed), 0); +} diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs new file mode 100644 index 00000000000..c85a88e90f7 --- /dev/null +++ b/vmm_core/state_unit/src/tests/start.rs @@ -0,0 +1,473 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for start with rollback. + +use crate::State; +use crate::StateRequest; +use crate::StateUnit; +use crate::StateUnits; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use mesh::Receiver; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use test_with_tracing::test; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +struct RecordingUnit { + started: Arc, + stopped: Arc, +} + +impl StateUnit for RecordingUnit { + async fn start(&mut self) { + self.started.store(true, Ordering::Relaxed); + } + + async fn stop(&mut self) { + self.stopped.store(true, Ordering::Relaxed); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for RecordingUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +/// Runs `unit`, but drops each start request after starting the unit, without +/// answering it, so that the start is uncertain while the unit still answers +/// later requests. +async fn run_unanswering_start( + mut unit: RecordingUnit, + mut recv: Receiver, +) -> RecordingUnit { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.start().await; + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit +} + +fn recording_unit(started: &Arc, stopped: &Arc) -> RecordingUnit { + RecordingUnit { + started: started.clone(), + stopped: stopped.clone(), + } +} + +fn unit_state(units: &StateUnits, name: &str) -> State { + units + .inner + .lock() + .units + .values() + .find(|unit| unit.name.as_ref() == name) + .unwrap() + .state +} + +#[async_test] +async fn an_unanswered_start_rolls_back_and_blocks_dependents(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let independent_started = Arc::new(AtomicBool::new(false)); + let independent_stopped = Arc::new(AtomicBool::new(false)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + RecordingUnit { + started: independent_started.clone(), + stopped: independent_stopped.clone(), + }, + recv, + ) + }) + .unwrap(); + // A unit whose task drops its start request without answering, as a + // task that has failed would. + let unanswered = units + .add("unanswered") + .spawn(&driver, |mut recv| async move { + while let Some(request) = recv.next().await { + if matches!(request, StateRequest::Start(_)) { + break; + } + } + }) + .unwrap(); + let dependent_started = Arc::new(AtomicBool::new(false)); + let _dependent = units + .add("dependent") + .depends_on(unanswered.handle()) + .spawn(&driver, |recv| { + run_unit( + RecordingUnit { + started: dependent_started.clone(), + stopped: Arc::new(AtomicBool::new(false)), + }, + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("unanswered"), "{error:#}"); + assert!(independent_started.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(!dependent_started.load(Ordering::Relaxed)); + // The unanswered unit may be running. + assert!(units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_uncertain_start_keeps_units_running_until_stopped(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let independent_started = Arc::new(AtomicBool::new(false)); + let independent_stopped = Arc::new(AtomicBool::new(false)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&independent_started, &independent_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!(error.contains("unanswered"), "{error}"); + assert!(independent_started.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!unanswered_stopped.load(Ordering::Relaxed)); + // The unanswered unit is running, so a stop must still be able to stop it. + assert!(units.is_running()); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + units.stop().await; + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_failed_hot_add_does_not_stop_dependency_of_running_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let running_dependent_started = Arc::new(AtomicBool::new(false)); + let running_dependent_stopped = Arc::new(AtomicBool::new(false)); + let running_dependent = units + .add("running-dependent") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&running_dependent_started, &running_dependent_stopped), + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let _dependency = units + .add("new-dependency") + .dependency_of(running_dependent.handle()) + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start_stopped_units().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(!running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!unanswered_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + units.stop().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_started_unit_exit_during_rollback_returns_error(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + // A unit whose task answers its start request and then exits. + let exited = units + .add("exited") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| async move { + let Some(StateRequest::Start(response)) = recv.next().await else { + panic!("unexpected request"); + }; + response.complete(()); + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop failed"), + "{error}" + ); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + // The dependency is still running, so a later stop must be able to stop it. + assert!(units.is_running()); + + exited.remove().await; + units.stop().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_uncertain_start_blocks_dependency_rollback(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .depends_on(dependency.handle()) + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + + units.stop().await; + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_initial_start_rollback_can_be_retried(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let started = Arc::new(AtomicBool::new(false)); + let stopped = Arc::new(AtomicBool::new(false)); + let stop_requests = Arc::new(AtomicUsize::new(0)); + // A unit that loses the response to its first stop request without + // stopping. + let _unit = units + .add("rollback-retry") + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&started, &stopped); + let stop_requests = stop_requests.clone(); + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop failed"), + "{error}" + ); + assert!(started.load(Ordering::Relaxed)); + assert!(!stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 1); + assert!(units.is_running()); + + units.stop().await; + assert!(stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); +} + +#[async_test] +async fn test_teardown_stop_after_unit_exits_before_answering_start(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + // A unit whose task answers a start and a stop, and then exits before it + // answers the next start, as a task that fails while the VM restarts after + // a failed save would. + let _exited = units + .add("exited") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| async move { + for _ in 0..2 { + match recv.next().await { + Some(StateRequest::Start(response)) => response.complete(()), + Some(StateRequest::Stop(response)) => response.complete(()), + _ => panic!("unexpected request"), + } + } + }) + .unwrap(); + units.start().await.unwrap(); + units.stop().await; + dependency_stopped.store(false, Ordering::Relaxed); + + let error = units.start().await.unwrap_err().to_string(); + assert!(error.contains("exited"), "{error}"); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + + // Tearing the VM down stops the units without the exited unit answering. + units.stop_for_teardown().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "exited"), State::StartUncertain); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); +} diff --git a/vmm_core/vmotherboard/Cargo.toml b/vmm_core/vmotherboard/Cargo.toml index b71f155d9e1..a834bf12c51 100644 --- a/vmm_core/vmotherboard/Cargo.toml +++ b/vmm_core/vmotherboard/Cargo.toml @@ -58,6 +58,7 @@ thiserror.workspace = true tracing.workspace = true [dev-dependencies] +inspect = { workspace = true, features = ["initiate"] } test_with_tracing.workspace = true [lints] diff --git a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs index 15783878967..155adf83ef3 100644 --- a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs +++ b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs @@ -9,16 +9,20 @@ use closeable_mutex::CloseableMutex; use futures::FutureExt; use futures::StreamExt; use futures::future::BoxFuture; +use futures::future::Fuse; use futures::task::ArcWake; use futures::task::WakerRef; use futures::task::waker_ref; use inspect::InspectMut; +use mesh::error::RemoteError; +use mesh::rpc::FailableRpc; use pal_async::task::Spawn; use state_unit::SpawnedUnit; use state_unit::StateRequest; use state_unit::StateUnit; use std::sync::Arc; use std::task::Context; +use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; use vmcore::save_restore::SavedStateBlob; @@ -97,6 +101,8 @@ impl InspectMut for ArcMutexChipsetDeviceUnit { trait DynDevice: InspectMut + Send { fn start(&mut self); async fn stop(&mut self); + fn quiesce_input(&mut self) -> InputHookCompletion; + fn resume_input(&mut self) -> InputHookCompletion; async fn reset(&mut self); fn poll_device(&mut self, cx: &mut Context<'_>); async fn save(&mut self) -> Result; @@ -113,6 +119,14 @@ impl DynDevice for T { self.stop().await } + fn quiesce_input(&mut self) -> InputHookCompletion { + self.quiesce_input() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.resume_input() + } + async fn reset(&mut self) { self.reset().await } @@ -166,12 +180,50 @@ impl ArcMutexChipsetDeviceUnit { } pub async fn run(mut self, mut state_change: mesh::Receiver) -> Self { + // An input hook request, whose answer waits for the hook's completion. + // The unit keeps polling the device meanwhile, since vCPUs may still + // access it and the completion may wait for the device to progress. + let mut input_hook: Option<(FailableRpc<(), ()>, InputHookCompletion)> = None; + // A state request that the unit has received but not applied. A + // transaction sends no request before the input hook is answered, but + // a dropped one can. Since only an inspection may overlap the hook, any + // other request waits for it, and the unit receives no other request + // until it has applied this one. + let mut request = None; + let mut closed = false; loop { + if let Some(req) = request + .take_if(|req| input_hook.is_none() || matches!(req, StateRequest::Inspect(_))) + { + match req { + StateRequest::QuiesceInput(rpc) => { + input_hook = Some((rpc, StateUnit::quiesce_input(&mut self))); + } + StateRequest::ResumeInput(rpc) => { + input_hook = Some((rpc, StateUnit::resume_input(&mut self))); + } + req => req.apply(&mut self).await, + } + continue; + } + if closed && input_hook.is_none() { + break; + } + enum Event<'a> { - StateChange(StateRequest), + StateChange(Option), Poll(WakerRef<'a>), + InputHookDone(anyhow::Result<()>), } + // Wait for state requests until the channel closes, but not while a + // request waits for the input hook. + let mut state_change_fut = if closed || request.is_some() { + Fuse::terminated() + } else { + state_change.next().fuse() + }; + // Wait for poll requests. let poll_fut = async { if self.running { @@ -183,27 +235,35 @@ impl ArcMutexChipsetDeviceUnit { std::future::pending().await }; + // Wait for the completion of the pending input hook. + let mut input_hook_fut = match &mut input_hook { + Some((_, completion)) => completion.fuse(), + None => Fuse::terminated(), + }; + let event = futures::select! { - req = state_change.next() => { - if let Some(req) = req { - Event::StateChange(req) - } else { - break; - } + req = state_change_fut => { + Event::StateChange(req) } waker = poll_fut.fuse() => { Event::Poll(waker) } + result = input_hook_fut => { + Event::InputHookDone(result) + } }; match event { - Event::StateChange(req) => { - req.apply(&mut self).await; - } + Event::StateChange(None) => closed = true, + Event::StateChange(Some(req)) => request = Some(req), Event::Poll(waker) => { let mut device = self.device.lock(); device.poll_device(&mut Context::from_waker(&waker)); } + Event::InputHookDone(result) => { + let (rpc, _) = input_hook.take().expect("input hook is pending"); + rpc.complete(result.map_err(RemoteError::new)); + } } } self @@ -234,6 +294,20 @@ impl StateUnit for ArcMutexChipsetDeviceUnit { // These are currently not true. } + fn quiesce_input(&mut self) -> InputHookCompletion { + // VPs are still running during input transitions, so keep the mutex + // open: a concurrent guest access waits for the hook, as for any other + // device call, instead of panicking on a closed mutex. The hook is + // synchronous, and the unit awaits its completion after the lock is + // released, so a guest access never waits for asynchronous work, which + // might need the executor that the guest access would block. + self.device.lock().quiesce_input() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.device.lock().resume_input() + } + async fn reset(&mut self) -> anyhow::Result<()> { self.device.clone().close().reset().await; Ok(()) @@ -264,21 +338,36 @@ mod tests { use chipset_device::io::IoResult; use chipset_device::mmio::MmioIntercept; use chipset_device::pio::PortIoIntercept; + use chipset_device::poll_device::PollDevice; use chipset_device_resources::ResolvedChipsetDevice; use closeable_mutex::CloseableMutex; + use futures::future::Either; use futures::poll; use inspect::InspectMut; + use mesh::rpc::FailableRpc; + use mesh::rpc::PendingFailableRpc; + use mesh::rpc::RpcSend; use pal_async::DefaultDriver; + use pal_async::DefaultPool; use pal_async::async_test; + use pal_async::task::Spawn; + use pal_async::timer::PolledTimer; + use state_unit::StateRequest; use state_unit::StateUnit; use state_unit::StateUnits; + use std::pin::pin; use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; + use std::sync::mpsc; + use std::task::Context; use std::task::Poll; + use std::task::Waker; + use std::time::Duration; use test_with_tracing::test; use vmcore::device_state::ChangeDeviceState; + use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::AsyncSaveRestore; use vmcore::save_restore::NoSavedState; use vmcore::save_restore::RestoreError; @@ -756,4 +845,324 @@ mod tests { } } } + + /// A device whose input hooks signal that they ran and complete only when + /// the test releases them. + struct InputHookDevice { + started: Option>, + release: Option>, + } + + impl InputHookDevice { + fn hook(&mut self) -> InputHookCompletion { + self.started.take().expect("hook runs once").send(()); + let release = self.release.take().expect("hook runs once"); + InputHookCompletion::new(async move { + release + .await + .map_err(|_| anyhow::anyhow!("hook release dropped")) + }) + } + } + + impl ChipsetDevice for InputHookDevice {} + + impl InspectMut for InputHookDevice { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } + } + + impl ChangeDeviceState for InputHookDevice { + fn start(&mut self) {} + + fn quiesce_input(&mut self) -> InputHookCompletion { + self.hook() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.hook() + } + + async fn stop(&mut self) {} + + async fn reset(&mut self) {} + } + + impl AsyncSaveRestore for InputHookDevice { + type SavedState = NoSavedState; + + async fn save(&mut self) -> Result { + Ok(NoSavedState) + } + + async fn restore(&mut self, _: NoSavedState) -> Result<(), RestoreError> { + Ok(()) + } + } + + /// Sends `request` to the unit of an [`InputHookDevice`] whose hook does not + /// complete until a guest access to the device, on the same single-threaded + /// executor, releases it, as when a VP and the unit share a CPU's thread in + /// OpenHCL's threadpool. + /// + /// The guest access locks the device as MMIO and PIO dispatch do. If the + /// hook held the lock until its completion, the guest access would block + /// the executor that must complete the hook, so this fails after a timeout + /// instead of hanging. + fn input_hook_with_guest_access_on_one_executor( + request: fn(FailableRpc<(), ()>) -> StateRequest, + ) { + let (done_send, done_recv) = mpsc::channel(); + std::thread::spawn(move || { + DefaultPool::run_with(async |driver| { + let (started_send, started_recv) = mesh::oneshot(); + let (release_send, release_recv) = mesh::oneshot(); + let device = Arc::new(CloseableMutex::new(InputHookDevice { + started: Some(started_send), + release: Some(release_recv), + })); + let unit = ArcMutexChipsetDeviceUnit::new(device.clone(), false); + let (send, recv) = mesh::channel(); + let unit_task = driver.spawn("unit", unit.run(recv)); + let guest_access = driver.spawn("guest-access", async move { + started_recv.await.unwrap(); + drop(device.lock_if_open().expect("device is open")); + release_send.send(()); + }); + send.call_failable(request, ()).await.unwrap(); + guest_access.await; + drop(send); + unit_task.await; + }); + done_send.send(()).unwrap(); + }); + done_recv + .recv_timeout(Duration::from_secs(10)) + .expect("a guest access deadlocked with an input hook on its executor"); + } + + #[test] + fn input_hooks_do_not_block_guest_access_on_their_executor() { + input_hook_with_guest_access_on_one_executor(StateRequest::QuiesceInput); + input_hook_with_guest_access_on_one_executor(StateRequest::ResumeInput); + } + + /// A device whose input hooks complete only once the unit polls the device + /// after the hook ran, as a device that finishes reading host input when it + /// is polled would. + #[derive(Default)] + struct PolledHookDevice { + waker: Option, + hook_started: Option>, + polled: Option>, + } + + impl PolledHookDevice { + fn hook(&mut self) -> InputHookCompletion { + if let Some(hook_started) = self.hook_started.take() { + hook_started.send(()); + } + let (send, recv) = mesh::oneshot(); + self.polled = Some(send); + InputHookCompletion::new(async move { + recv.await + .map_err(|_| anyhow::anyhow!("poll signal dropped")) + }) + } + } + + impl ChipsetDevice for PolledHookDevice { + fn supports_poll_device(&mut self) -> Option<&mut dyn PollDevice> { + Some(self) + } + } + + impl PollDevice for PolledHookDevice { + fn poll_device(&mut self, cx: &mut Context<'_>) { + self.waker = Some(cx.waker().clone()); + if let Some(polled) = self.polled.take() { + polled.send(()); + } + } + } + + impl InspectMut for PolledHookDevice { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } + } + + impl ChangeDeviceState for PolledHookDevice { + fn start(&mut self) {} + + fn quiesce_input(&mut self) -> InputHookCompletion { + self.hook() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.hook() + } + + async fn stop(&mut self) {} + + async fn reset(&mut self) {} + } + + impl AsyncSaveRestore for PolledHookDevice { + type SavedState = NoSavedState; + + async fn save(&mut self) -> Result { + Ok(NoSavedState) + } + + async fn restore(&mut self, _: NoSavedState) -> Result<(), RestoreError> { + Ok(()) + } + } + + /// Starts the unit of a new [`PolledHookDevice`], with the device. + async fn start_polled_hook_unit( + driver: &DefaultDriver, + ) -> ( + Arc>, + mesh::Sender, + pal_async::task::Task, + ) { + let device = Arc::new(CloseableMutex::new(PolledHookDevice::default())); + let unit = ArcMutexChipsetDeviceUnit::new(device.clone(), false); + let (send, recv) = mesh::channel(); + let unit_task = driver.spawn("unit", unit.run(recv)); + send.call(StateRequest::Start, ()).await.unwrap(); + (device, send, unit_task) + } + + /// Sends `request` to the unit of `device`, without waiting for its answer, + /// and returns once the device's hook has run. + async fn start_polled_hook( + device: &CloseableMutex, + send: &mesh::Sender, + request: fn(FailableRpc<(), ()>) -> StateRequest, + ) -> PendingFailableRpc<()> { + let (started_send, started_recv) = mesh::oneshot(); + device.lock().hook_started = Some(started_send); + let hook = send.call_failable(request, ()); + started_recv.await.unwrap(); + hook + } + + /// Asks the unit of `device` to poll it. + fn request_poll(device: &CloseableMutex) { + device + .lock() + .waker + .as_ref() + .expect("device was polled") + .wake_by_ref(); + } + + /// Waits for `fut`, failing after a timeout with `message`. + async fn with_timeout( + driver: &DefaultDriver, + fut: impl Future, + message: &str, + ) -> T { + let mut timer = PolledTimer::new(driver); + let timeout = timer.sleep(Duration::from_secs(10)); + match futures::future::select(pin!(fut), pin!(timeout)).await { + Either::Left((output, _)) => output, + Either::Right(_) => panic!("{message}"), + } + } + + #[async_test] + async fn input_hooks_keep_polling_the_device(driver: DefaultDriver) { + let (device, send, unit_task) = start_polled_hook_unit(&driver).await; + + let requests: [fn(FailableRpc<(), ()>) -> StateRequest; 2] = + [StateRequest::QuiesceInput, StateRequest::ResumeInput]; + for request in requests { + let hook = start_polled_hook(&device, &send, request).await; + request_poll(&device); + with_timeout( + &driver, + hook, + "the unit did not poll the device during an input hook", + ) + .await + .unwrap(); + } + + send.call(StateRequest::Stop, ()).await.unwrap(); + drop(send); + unit_task.await; + } + + #[async_test] + async fn queued_request_waits_for_a_polled_input_hook(driver: DefaultDriver) { + let (device, send, unit_task) = start_polled_hook_unit(&driver).await; + + // Drop the answer to the hook, as a dropped input transaction would, + // and stop the unit while the hook is pending. + drop(start_polled_hook(&device, &send, StateRequest::QuiesceInput).await); + let mut stop = pin!(send.call(StateRequest::Stop, ())); + PolledTimer::new(&driver) + .sleep(Duration::from_millis(100)) + .await; + assert!(poll!(stop.as_mut()).is_pending()); + + // The stop waits for the hook, which the unit completes when it polls + // the device. + request_poll(&device); + with_timeout( + &driver, + stop, + "the unit did not poll the device before a stop", + ) + .await + .unwrap(); + drop(send); + unit_task.await; + } + + #[async_test] + async fn closed_channel_waits_for_a_polled_input_hook(driver: DefaultDriver) { + let (device, send, unit_task) = start_polled_hook_unit(&driver).await; + + // Drop the answer to the hook and close the unit's channel, as a + // removal that follows a dropped input transaction would. + drop(start_polled_hook(&device, &send, StateRequest::QuiesceInput).await); + drop(send); + PolledTimer::new(&driver) + .sleep(Duration::from_millis(100)) + .await; + + // The unit exits once the hook completes, when it polls the device. + request_poll(&device); + with_timeout( + &driver, + unit_task, + "the unit did not poll the device before exiting", + ) + .await; + } + + #[async_test] + async fn inspection_does_not_wait_for_a_polled_input_hook(driver: DefaultDriver) { + let (device, send, unit_task) = start_polled_hook_unit(&driver).await; + + let hook = start_polled_hook(&device, &send, StateRequest::QuiesceInput).await; + let mut inspection = inspect::inspect("", &inspect::send(&send, StateRequest::Inspect)); + with_timeout( + &driver, + inspection.resolve(), + "the inspection waited for the input hook", + ) + .await; + + request_poll(&device); + hook.await.unwrap(); + drop(send); + unit_task.await; + } } diff --git a/workers/chipset_device_worker/src/protocol.rs b/workers/chipset_device_worker/src/protocol.rs index 52ab0e9dd0b..3a479f9dc5b 100644 --- a/workers/chipset_device_worker/src/protocol.rs +++ b/workers/chipset_device_worker/src/protocol.rs @@ -61,6 +61,10 @@ pub(crate) enum DeviceRequest { PciConfigWrite(WriteRequest, PciConfigByteEnable), /// Start the device Start, + /// Stop accepting new host input before snapshot. + QuiesceInput(mesh::rpc::FailableRpc<(), ()>), + /// Resume host input after a failed snapshot. + ResumeInput(mesh::rpc::FailableRpc<(), ()>), /// Stop the device Stop(Rpc<(), ()>), /// Reset the device diff --git a/workers/chipset_device_worker/src/proxy.rs b/workers/chipset_device_worker/src/proxy.rs index 05ded9079da..119ef81b38f 100644 --- a/workers/chipset_device_worker/src/proxy.rs +++ b/workers/chipset_device_worker/src/proxy.rs @@ -26,12 +26,14 @@ use chipset_device_resources::ResolveChipsetDeviceHandleParams; use futures::StreamExt; use inspect::Inspect; use inspect::InspectMut; +use mesh::rpc::FailableRpc; use mesh::rpc::RpcSend; use mesh_worker::WorkerHandle; use slab::Slab; use std::task::Poll; use task_control::TaskControl; use vmcore::device_state::ChangeDeviceState; +use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::AsyncProtobufSaveRestore; use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; @@ -130,6 +132,13 @@ impl ChipsetDeviceProxy { _guest_memory: guest_memory, }) } + + /// Sends an input hook request to the remote device, and returns a + /// completion that waits for its answer. + fn input_hook(&self, request: fn(FailableRpc<(), ()>) -> DeviceRequest) -> InputHookCompletion { + let call = self.req_send.call_failable(request, ()); + InputHookCompletion::new(async move { Ok(call.await?) }) + } } impl ChipsetDevice for ChipsetDeviceProxy { @@ -275,6 +284,14 @@ impl ChangeDeviceState for ChipsetDeviceProxy { self.req_send.send(DeviceRequest::Start) } + fn quiesce_input(&mut self) -> InputHookCompletion { + self.input_hook(DeviceRequest::QuiesceInput) + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.input_hook(DeviceRequest::ResumeInput) + } + async fn stop(&mut self) { self.req_send .call(DeviceRequest::Stop, ()) @@ -317,6 +334,7 @@ impl AsyncProtobufSaveRestore for ChipsetDeviceProxy { #[cfg(test)] mod tests { use super::ChipsetDeviceProxy; + use super::PioProxy; use crate::RemoteDynamicResolvers; use crate::guestmem::GuestMemoryTask; use crate::protocol::DeviceInit; @@ -324,17 +342,25 @@ mod tests { use crate::worker::RemoteChipsetDeviceWorker; use crate::worker::RemoteChipsetDeviceWorkerParameters; use chipset_device::ChipsetDevice; + use chipset_device::io::IoResult; + use chipset_device::pio::PortIoIntercept; + use chipset_device::poll_device::PollDevice; use chipset_device_resources::ResolveChipsetDeviceHandleParams; use chipset_device_resources::ResolvedChipsetDevice; + use futures::future::Either; use guestmem::GuestMemory; use inspect::InspectMut; use mesh::MeshPayload; use mesh::payload::Protobuf; use pal_async::DefaultDriver; use pal_async::async_test; + use pal_async::timer::PolledTimer; use slab::Slab; use std::convert::Infallible; + use std::pin::Pin; + use std::pin::pin; use std::task::Poll; + use std::time::Duration; use task_control::TaskControl; use test_with_tracing::test; use vm_resource::IntoResource; @@ -343,6 +369,7 @@ mod tests { use vm_resource::ResourceResolver; use vm_resource::kind::ChipsetDeviceHandleKind; use vmcore::device_state::ChangeDeviceState; + use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::AsyncProtobufSaveRestore; use vmcore::save_restore::AsyncSaveRestore; use vmcore::save_restore::RestoreError; @@ -366,6 +393,8 @@ mod tests { guest_memory: GuestMemory, #[inspect(skip)] encrypted_guest_memory: GuestMemory, + #[inspect(skip)] + port_write: Option>, } impl TestDevice { @@ -382,12 +411,51 @@ mod tests { assert_eq!(actual, expected); } } + + /// Accesses guest memory, and returns a completion that waits for the + /// next guest port write. + fn input_hook(&mut self) -> InputHookCompletion { + self.access_memory(); + let (send, recv) = mesh::oneshot(); + self.port_write = Some(send); + InputHookCompletion::new(async move { + recv.await + .map_err(|_| anyhow::anyhow!("port write signal dropped")) + }) + } } - impl ChipsetDevice for TestDevice {} + impl ChipsetDevice for TestDevice { + fn supports_pio(&mut self) -> Option<&mut dyn PortIoIntercept> { + Some(self) + } + } + + impl PortIoIntercept for TestDevice { + fn io_read(&mut self, _io_port: u16, data: &mut [u8]) -> IoResult { + data.fill(0); + IoResult::Ok + } + + fn io_write(&mut self, _io_port: u16, _data: &[u8]) -> IoResult { + if let Some(port_write) = self.port_write.take() { + port_write.send(()); + } + IoResult::Ok + } + } impl ChangeDeviceState for TestDevice { fn start(&mut self) {} + + fn quiesce_input(&mut self) -> InputHookCompletion { + self.input_hook() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.input_hook() + } + async fn stop(&mut self) { self.access_memory(); } @@ -456,6 +524,7 @@ mod tests { value: 0, guest_memory: input.guest_memory.clone(), encrypted_guest_memory: input.encrypted_guest_memory.clone(), + port_write: None, }; device.access_memory(); Ok(device.into()) @@ -506,7 +575,7 @@ mod tests { .unwrap(); let DeviceInit { mmio, pio, pci } = cap_recv.await.unwrap(); - assert!(mmio.is_none() && pio.is_none() && pci.is_none()); + assert!(mmio.is_none() && pci.is_none()); ChipsetDeviceProxy { req_send, resp_recv, @@ -515,7 +584,9 @@ mod tests { in_flight_writes: Slab::new(), _guest_memory: guest_memory, mmio: None, - pio: None, + pio: pio.map(|_| PioProxy { + static_regions: Vec::new(), + }), pci: None, } } @@ -590,4 +661,49 @@ mod tests { ); stop_worker(&mut proxy).await; } + + /// Runs `hook` on the proxy of a device that completes the hook only after a + /// guest port write, and writes to the port while the hook is pending, as a + /// vCPU may. This fails after a timeout if the worker stops serving guest + /// requests, or the proxy stops completing them, while the hook is pending. + async fn input_hook_with_guest_write( + driver: &DefaultDriver, + proxy: &mut ChipsetDeviceProxy, + hook: fn(&mut ChipsetDeviceProxy) -> InputHookCompletion, + ) { + let mut completion = hook(proxy); + let IoResult::Defer(mut write) = proxy.io_write(0, &[1]) else { + panic!("proxy port writes are deferred"); + }; + let mut written = false; + let round_trip = std::future::poll_fn(|cx| { + proxy.poll_device(cx); + if !written && let Poll::Ready(result) = write.poll_write(cx) { + result.unwrap(); + written = true; + } + if written { + Pin::new(&mut completion).poll(cx) + } else { + Poll::Pending + } + }); + let mut timer = PolledTimer::new(driver); + let timeout = timer.sleep(Duration::from_secs(10)); + match futures::future::select(pin!(round_trip), pin!(timeout)).await { + Either::Left((result, _)) => result.unwrap(), + Either::Right(_) => panic!("a guest request was not served during an input hook"), + } + } + + #[async_test] + async fn input_hooks_serve_guest_requests(driver: DefaultDriver) { + let vmtime = VmTimeKeeper::new(&driver, VmTime::from_100ns(0)); + let mut proxy = new_proxy(&vmtime, &driver).await; + + input_hook_with_guest_write(&driver, &mut proxy, ChangeDeviceState::quiesce_input).await; + input_hook_with_guest_write(&driver, &mut proxy, ChangeDeviceState::resume_input).await; + + stop_worker(&mut proxy).await; + } } diff --git a/workers/chipset_device_worker/src/worker.rs b/workers/chipset_device_worker/src/worker.rs index e3b90a96c17..572f61ed2ad 100644 --- a/workers/chipset_device_worker/src/worker.rs +++ b/workers/chipset_device_worker/src/worker.rs @@ -24,15 +24,19 @@ use chipset_device_resources::ErasedChipsetDevice; use chipset_device_resources::ResolveChipsetDeviceHandleParams; use mesh::MeshPayload; use mesh::error::RemoteError; +use mesh::rpc::FailableRpc; use mesh_worker::Worker; use mesh_worker::WorkerId; use mesh_worker::WorkerRpc; use pal_async::DefaultPool; +use std::future::Future; +use std::pin::Pin; use std::task::Poll; use vm_resource::Resource; use vm_resource::ResourceResolver; use vm_resource::kind::ChipsetDeviceHandleKind; use vmcore::device_state::ChangeDeviceState; +use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::AsyncProtobufSaveRestore; /// Worker ID for ChipsetDevice workers. @@ -190,6 +194,10 @@ impl Worker for RemoteChipsetDeviceWorker { fn run(mut self, mut rpc_recv: mesh::Receiver>) -> anyhow::Result<()> { self.pool.take().unwrap().run_until(async move { + // An input hook request, whose answer waits for the hook's + // completion. The worker keeps serving guest requests and polling + // the device meanwhile, since vCPUs may still access the device. + let mut input_hook: Option<(FailableRpc<(), ()>, InputHookCompletion)> = None; loop { enum WorkerEvent { Rpc(WorkerRpc<()>), @@ -203,6 +211,13 @@ impl Worker for RemoteChipsetDeviceWorker { poll_device.poll_device(cx); } + if let Some((_, completion)) = &mut input_hook + && let Poll::Ready(result) = Pin::new(completion).poll(cx) + { + let (rpc, _) = input_hook.take().unwrap(); + rpc.complete(result.map_err(RemoteError::new)); + } + self.deferred_reads .extract_if(.., |read| { let mut data = vec![0; read.size]; @@ -265,6 +280,12 @@ impl Worker for RemoteChipsetDeviceWorker { self.device.start(); self.running = true; } + DeviceRequest::QuiesceInput(rpc) => { + input_hook = Some((rpc, self.device.quiesce_input())); + } + DeviceRequest::ResumeInput(rpc) => { + input_hook = Some((rpc, self.device.resume_input())); + } DeviceRequest::Teardown(rpc) => { assert!( !self.running,