Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 20 additions & 9 deletions openhcl/underhill_core/src/dispatch/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc;

#[derive(MeshPayload)]
pub enum UhVmRpc {
Pause(Rpc<(), bool>),
Pause(FailableRpc<(), bool>),
Resume(Rpc<(), bool>),
Save(FailableRpc<(), Vec<u8>>),
ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume
Expand Down Expand Up @@ -326,7 +326,15 @@ impl LoadedVm {
WorkerRpc::Stop => break None,
WorkerRpc::Restart(rpc) => {
let state = async {
let running = self.stop().await;
let running = match self.stop().await {
Ok(running) => running,
Err(err) => {
rpc.complete(Err(RemoteError::new(
err.context("failed to stop VM for worker restart"),
)));
return None;
}
};
match self
.save(
SaveReason::WorkerRestart,
Expand Down Expand Up @@ -416,10 +424,10 @@ impl LoadedVm {
})
.await
}
UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await,
UhVmRpc::Pause(rpc) => rpc.handle_failable(async |()| self.stop().await).await,
UhVmRpc::Save(rpc) => {
rpc.handle_failable(async |()| {
let running = self.stop().await;
let running = self.stop().await?;
let r = self
.save(
SaveReason::Diagnostic,
Expand Down Expand Up @@ -725,7 +733,7 @@ impl LoadedVm {

// Do everything before the log flush under a span.
let r = async {
if !self.stop().await {
if !self.stop().await? {
// This should only occur if you tried to initiate a
// servicing operation after manually pausing underhill
// via `ohcldiag-dev`.
Expand Down Expand Up @@ -921,14 +929,17 @@ impl LoadedVm {
}

/// Returns true if the VM was previously running.
async fn stop(&mut self) -> bool {
///
/// Fails if a state unit does not stop. The state units then stay
/// running, and can never start again.
async fn stop(&mut self) -> anyhow::Result<bool> {
if self.state_units.is_running() {
self.last_state_unit_stop = Some(ReferenceTime::new(self.partition.reference_time()));
tracing::info!(CVM_ALLOWED, "stopping VM");
self.state_units.stop().await;
true
self.state_units.stop().await?;
Ok(true)
} else {
false
Ok(false)
}
}

Expand Down
8 changes: 7 additions & 1 deletion openhcl/underhill_core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,13 @@ async fn run_control(
// FUTURE: consider supporting cancellation
driver
.spawn("diag-pause", async move {
let was_paused = req.await.expect("failed to pause VM");
let was_paused = match req.await.expect("failed to pause VM") {
Ok(was_paused) => was_paused,
Err(error) => {
rpc.complete(Err(error));
return;
}
};
rpc.handle_failable_sync(|_| {
if !was_paused {
Err(anyhow::anyhow!("VM is already paused"))
Expand Down
69 changes: 57 additions & 12 deletions openvmm/openvmm_core/src/worker/dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4018,13 +4018,24 @@ impl LoadedVm {
self.state_units.start().await
}

async fn pause(&mut self) -> bool {
/// Pauses the VM, returning true if it was running.
///
/// Fails if a state unit does not stop. The VM no longer runs then, but
/// state units still may, which [`Self::units_left_running`] reports.
async fn pause(&mut self) -> anyhow::Result<bool> {
if !self.running {
return false;
return Ok(false);
}
self.state_units.stop().await;
self.running = false;
true
self.state_units.stop().await?;
Ok(true)
}

/// Returns true if state units run while the VM does not, as a failed
/// stop, or a start whose rollback did not stop every unit, leaves them.
/// They can never start again, so the VM must be torn down.
fn units_left_running(&self) -> bool {
self.state_units.is_running() && !self.running
}

/// Assign PCI bus numbers and BAR addresses for all boot modes.
Expand Down Expand Up @@ -4067,10 +4078,16 @@ impl LoadedVm {
.context("PCI resource assignment failed");

// Stop state units again; the caller will resume normally.
self.state_units.stop().await;
let stopped = self
.state_units
.stop()
.await
.context("failed to stop devices after PCI resource assignment");
drop(stop_guard);

result
// Devices that did not stop can never start again, which outweighs a
// failed assignment.
stopped.and(result)
}

pub async fn run(
Expand Down Expand Up @@ -4111,6 +4128,13 @@ impl LoadedVm {
let mut deferred = VecDeque::new();

loop {
// A failed stop, or a start whose rollback did not stop every
// unit, leaves state units running while the VM is paused. They
// can never start again, so tear the VM down.
if self.units_left_running() {
tracing::error!("state units failed to stop; tearing down the VM");
break;
}
let queued = if !device_removal.is_pending() {
deferred.pop_front()
} else {
Expand Down Expand Up @@ -4157,14 +4181,18 @@ impl LoadedVm {
WorkerRpc::Stop => break,
WorkerRpc::Restart(rpc) => {
let mut stopped = false;
let mut stop_failed = false;
// First run the non-destructive operations.
let r = async {
let shared_memory = self.inner.memory_manager.shared_memory_backing();
if shared_memory.is_none() {
anyhow::bail!("restart is not supported with --private-memory");
}
if self.running {
self.state_units.stop().await;
if let Err(error) = self.state_units.stop().await {
stop_failed = true;
return Err(error);
}
stopped = true;
}
let saved_state = self.save().await?;
Expand All @@ -4180,6 +4208,14 @@ impl LoadedVm {
return;
}
Err(err) => {
if stop_failed {
// The state units that did not stop can
// never start again, so tear the VM down.
rpc.complete(Err(RemoteError::new(err.context(
"worker restart failed and the VM could not stop",
))));
break;
}
if stopped {
if let Err(start_error) = self.start_units().await {
rpc.complete(Err(RemoteError::new(start_error.context(
Expand Down Expand Up @@ -4220,7 +4256,9 @@ impl LoadedVm {
VmRpc::Resume(rpc) => {
rpc.handle_failable(async |()| self.resume().await).await
}
VmRpc::Pause(rpc) => rpc.handle(async |()| self.pause().await).await,
VmRpc::Pause(rpc) => {
rpc.handle_failable(async |()| self.pause().await).await
}
VmRpc::Save(rpc) => {
rpc.handle_failable(async |()| self.save().await.map(ProtobufMessage::new))
.await
Expand Down Expand Up @@ -4324,7 +4362,7 @@ impl LoadedVm {
if !self.inner.partition.supports_reset() {
return Err(PulseSaveRestoreError::ResetNotSupported);
}
let paused = self.pause().await;
let paused = self.pause().await?;
self.save_reset_restore().await?;

if paused {
Expand Down Expand Up @@ -4630,9 +4668,16 @@ impl LoadedVm {
device_removal.drain().await;
// Stop the state units whenever they run, not only when the VM does:
// after a failed state change, such as a restart that could not resume
// the VM, the two can disagree.
// the VM, or a stop that a unit did not answer, the two can disagree.
// The VM is torn down anyway, so a unit that does not stop only leaves
// its dependencies running.
if self.state_units.is_running() {
self.state_units.stop().await;
if let Err(error) = self.state_units.stop().await {
tracing::error!(
error = error.as_ref() as &dyn std::error::Error,
"failed to stop the state units before teardown"
);
}
}
self.running = false;
self.teardown_chipset_devices().await;
Expand Down Expand Up @@ -4840,7 +4885,7 @@ impl LoadedVm {
}

async fn reset(&mut self, reload_firmware: bool) -> anyhow::Result<()> {
let resume = self.pause().await;
let resume = self.pause().await?;

self.state_units.reset().await?;
// TODO: _vmnic
Expand Down
2 changes: 1 addition & 1 deletion openvmm/openvmm_core/src/worker/dispatch/dump.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ impl LoadedVm {
/// Pauses the VM if running, collects VP state and streams memory,
/// then restores the prior running state.
pub(super) async fn dump_state(&mut self, file: File) -> anyhow::Result<()> {
let was_running = self.pause().await;
let was_running = self.pause().await?;
let result = self.dump_state_inner(file).await;
if was_running {
self.resume()
Expand Down
28 changes: 19 additions & 9 deletions openvmm/openvmm_core/src/worker/dispatch/microvm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -319,9 +319,11 @@ impl LoadedVm {
///
/// The quiesce stops every state unit and has no deadline: a unit that
/// never stops stalls the capture until the process that drives the VM
/// ends it, with nothing published and the source never resumed. Each
/// error after the quiesce leaves every unit stopped, so the VM can roll
/// back with [`Self::resume_after_failed_snapshot`].
/// ends it, with nothing published and the source never resumed. A unit
/// whose task does not answer fails the quiesce instead: it can never
/// start again, so the VM can neither roll back nor run, and the worker
/// tears it down. Each error after the quiesce leaves every unit stopped,
/// so the VM can roll back with [`Self::resume_after_failed_snapshot`].
pub(super) async fn quiesce_for_snapshot(
&mut self,
) -> Result<openvmm_defs::rpc::SnapshotSaveResponse, openvmm_defs::rpc::SnapshotQuiesceError>
Expand Down Expand Up @@ -349,9 +351,13 @@ impl LoadedVm {
}

let quiesce = openvmm_defs::profile::ProfileSpan::start();
self.state_units.stop().await;
quiesce.complete("capture", "quiesce", Default::default());
// The VM no longer runs even if a unit does not stop, which leaves
// the units running for the worker to tear down.
self.running = false;
self.state_units.stop().await.map_err(|error| {
openvmm_defs::rpc::SnapshotQuiesceError::Unrecoverable(RemoteError::new(error))
})?;
quiesce.complete("capture", "quiesce", Default::default());

// The time ABI's capture checks and records: the LAPIC timers, then
// the capture anchor and records. The PIT checks itself when saved.
Expand Down Expand Up @@ -423,7 +429,9 @@ impl LoadedVm {
/// that runs a quiesced VM again.
///
/// A unit whose task does not answer leaves the units that did start
/// stopped again, and fails the rollback.
/// stopped again, and fails the rollback. A unit that such a unit depends
/// on, or that does not answer its stop, stays running instead, and the
/// worker then tears the VM down.
pub(super) async fn resume_after_failed_snapshot(&mut self) -> anyhow::Result<()> {
self.state_units.start().await?;
self.running = true;
Expand All @@ -437,9 +445,11 @@ impl LoadedVm {
match event {
Ok(request) => {
if !self.establish_snapshot_boundary(request).await {
if self.running {
self.state_units.stop().await;
self.running = false;
if let Err(error) = self.pause().await {
tracing::error!(
error = error.as_ref() as &dyn std::error::Error,
"failed to stop the VM after its snapshot boundary failed"
);
}
return false;
}
Expand Down
17 changes: 13 additions & 4 deletions openvmm/openvmm_core/src/worker/dispatch/restore.rs
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,14 @@ impl LoadedVm {
}
});
if let Err(error) = signal_result {
self.state_units.stop().await;
// A unit that does not stop stays running while the VM is
// paused, and the worker then tears the VM down.
if let Err(stop_error) = self.state_units.stop().await {
tracing::error!(
error = stop_error.as_ref() as &dyn std::error::Error,
"failed to stop the state units after the restore readiness event failed"
);
}
return Err(error).context("failed to publish restore readiness event");
}
}
Expand All @@ -367,9 +374,11 @@ impl LoadedVm {
/// input gate in time.
pub(super) async fn handle_restore_gate_timeout(&mut self) {
tracing::error!("post-restore input gate acknowledgement timed out");
if self.running {
self.state_units.stop().await;
self.running = false;
if let Err(error) = self.pause().await {
tracing::error!(
error = error.as_ref() as &dyn std::error::Error,
"failed to stop the VM after the post-restore input gate timed out"
);
}
}
}
Expand Down
11 changes: 6 additions & 5 deletions openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,11 @@ pub(super) fn filter(message: VmRpc, boundary_active: bool) -> Option<VmRpc> {
VmRpc::AddVpciDevice(rpc) => rpc.fail(SnapshotBoundaryActive),
VmRpc::RemoveVpciDevice(rpc) => rpc.fail(SnapshotBoundaryActive),
VmRpc::DumpState(rpc) => rpc.fail(SnapshotBoundaryActive),
VmRpc::Pause(rpc) => rpc.fail(SnapshotBoundaryActive),
VmRpc::PulseSaveRestore(rpc) => rpc.complete(Err(PulseSaveRestoreError::Other(
RemoteError::new(SnapshotBoundaryActive),
))),
VmRpc::Pause(rpc) | VmRpc::ClearHalt(rpc) => drop(rpc),
VmRpc::ClearHalt(rpc) => drop(rpc),
VmRpc::Nmi(rpc) => drop(rpc),
}
tracelimit::warn_ratelimited!(
Expand Down Expand Up @@ -190,16 +191,18 @@ mod tests {
let write = send.call(VmRpc::WriteMemory, (0, vec![1]));
let reset = send.call(VmRpc::Reset, ());
let resume = send.call(VmRpc::Resume, ());
let pause = send.call(VmRpc::Pause, ());
let remove = send.call(VmRpc::RemovePcieDevice, "port".to_owned());

for _ in 0..4 {
for _ in 0..5 {
assert!(filter(recv.try_recv().unwrap(), true).is_none());
}
for result in [block_on(write), block_on(reset), block_on(remove)] {
let error = result.unwrap().unwrap_err();
assert!(error.to_string().contains("snapshot boundary is active"));
}
assert!(block_on(resume).unwrap().is_err());
assert!(block_on(pause).unwrap().is_err());

let release = send.call(VmRpc::ReleaseSnapshotBoundary, ());
let Some(VmRpc::ReleaseSnapshotBoundary(rpc)) = filter(recv.try_recv().unwrap(), true)
Expand All @@ -213,13 +216,11 @@ mod tests {
#[test]
fn infallible_mutations_report_channel_error() {
let (send, mut recv) = mesh::channel();
let pause = send.call(VmRpc::Pause, ());
let clear_halt = send.call(VmRpc::ClearHalt, ());
let nmi = send.call(VmRpc::Nmi, 0);
for _ in 0..3 {
for _ in 0..2 {
assert!(filter(recv.try_recv().unwrap(), true).is_none());
}
assert!(block_on(pause).is_err());
assert!(block_on(clear_halt).is_err());
assert!(block_on(nmi).is_err());
}
Expand Down
10 changes: 9 additions & 1 deletion openvmm/openvmm_defs/src/rpc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,11 @@ pub enum VmRpc {
/// Release a post-OUT boundary without starting a snapshot transaction.
ReleaseSnapshotBoundary(FailableRpc<(), ()>),
Resume(FailableRpc<(), bool>),
Pause(Rpc<(), bool>),
/// Pause the VM, returning whether it was running.
///
/// Fails if a state unit does not stop; the worker then tears the VM
/// down, since that unit can never start again.
Pause(FailableRpc<(), bool>),
ClearHalt(Rpc<(), bool>),
Reset(FailableRpc<(), ()>),
Nmi(Rpc<u32, ()>),
Expand Down Expand Up @@ -87,6 +91,10 @@ pub enum SnapshotQuiesceError {
/// The VM quiesced completely; the controller may request rollback.
#[error("snapshot failed after the VM quiesced")]
RollbackSafe(#[source] RemoteError),
/// A state unit did not stop, so the VM can neither roll back nor run
/// again; the controller terminates the source.
#[error("snapshot quiesce left the VM unable to run again")]
Unrecoverable(#[source] RemoteError),
}

#[derive(Debug, MeshPayload, thiserror::Error)]
Expand Down
Loading
Loading