Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 101 additions & 45 deletions openhcl/underhill_core/src/dispatch/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc;
#[derive(MeshPayload)]
pub enum UhVmRpc {
Pause(Rpc<(), bool>),
Resume(Rpc<(), bool>),
Resume(FailableRpc<(), bool>),
Save(FailableRpc<(), Vec<u8>>),
ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume
PacketCapture(FailableRpc<PacketCaptureParams<Socket>, PacketCaptureParams<Socket>>),
Expand Down Expand Up @@ -226,6 +226,19 @@ pub struct LoadedVmState<T> {
pub control_send: mesh::Sender<ControlRequest>,
}

/// The outcome of a worker restart request.
#[expect(clippy::large_enum_variant)]
enum RestartOutcome<T> {
/// The VM state was saved, so the worker can restart.
Restart(FailableRpc<(), T>, ServicingState),
/// The restart failed, but the VM is back in its previous state, so the
/// worker can keep serving.
Recovered,
/// The restart failed and the VM could not be resumed, so the worker
/// cannot serve any more.
Unrecoverable,
}

impl LoadedVm {
/// Start running the VM which will start running VTL0.
pub async fn run<T: 'static + MeshPayload + Send>(
Expand All @@ -235,9 +248,12 @@ impl LoadedVm {
correlation_id: Option<Guid>,
mut vm_rpc: mesh::Receiver<UhVmRpc>,
mut worker_rpc: mesh::Receiver<WorkerRpc<T>>,
) -> Option<LoadedVmState<T>> {
) -> anyhow::Result<Option<LoadedVmState<T>>> {
if autostart_vps {
self.start(correlation_id).await;
if let Err(error) = self.start(correlation_id).await {
self.teardown().await;
return Err(error.context("failed to autostart VM"));
}
}

// VTL2 settings services
Expand Down Expand Up @@ -325,7 +341,7 @@ impl LoadedVm {
Event::WorkerRpc(message) => match message {
WorkerRpc::Stop => break None,
WorkerRpc::Restart(rpc) => {
let state = async {
let outcome = async {
let running = self.stop().await;
match self
.save(
Expand All @@ -336,26 +352,40 @@ impl LoadedVm {
)
.await
{
Ok(servicing_state) => Some((rpc, servicing_state)),
Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state),
Err(err) => {
if running {
self.start(None).await;
if let Err(start_error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
start_error.context(format!(
"worker restart failed ({err:#}) and the VM could not resume"
)),
)));
return RestartOutcome::Unrecoverable;
}
}
rpc.complete(Err(RemoteError::new(err)));
None
RestartOutcome::Recovered
}
}
}
.instrument(tracing::info_span!("restart", CVM_ALLOWED))
.await;

if let Some((rpc, servicing_state)) = state {
break Some(LoadedVmState {
restart_rpc: rpc,
servicing_state,
vm_rpc,
control_send: self.control_send.lock().take().unwrap(),
});
match outcome {
RestartOutcome::Restart(rpc, servicing_state) => {
break Some(LoadedVmState {
restart_rpc: rpc,
servicing_state,
vm_rpc,
control_send: self.control_send.lock().take().unwrap(),
});
}
RestartOutcome::Recovered => {}
// The error has already completed the RPC. Tear
// down through the common path rather than keep a
// worker whose state units can no longer run.
RestartOutcome::Unrecoverable => break None,
}
}
WorkerRpc::Inspect(deferred) => deferred.respond(|resp| {
Expand Down Expand Up @@ -406,34 +436,39 @@ impl LoadedVm {
}
Event::UhVmRpc(msg) => match msg {
UhVmRpc::Resume(rpc) => {
rpc.handle(async |()| {
if !self.state_units.is_running() {
self.start(None).await;
true
} else {
false
}
})
.await
if self.state_units.is_running() {
rpc.complete(Ok(false));
} else if let Err(error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
error.context("failed to resume VM"),
)));
// A failed start may leave state units terminal;
// this worker cannot safely continue serving.
break None;
} else {
rpc.complete(Ok(true));
}
}
UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await,
UhVmRpc::Save(rpc) => {
rpc.handle_failable(async |()| {
let running = self.stop().await;
let r = self
.save(
SaveReason::Diagnostic,
None,
KeepAliveConfig::Disabled,
KeepAliveConfig::Disabled,
)
.await;
if running {
self.start(None).await;
}
r.map(mesh::payload::encode)
})
.await
let running = self.stop().await;
let result = self
.save(
SaveReason::Diagnostic,
None,
KeepAliveConfig::Disabled,
KeepAliveConfig::Disabled,
)
.await;
if running && let Err(error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
error.context("failed to resume VM after save"),
)));
// The state units rolled back and cannot run
// again, so tear down rather than keep this worker.
break None;
}
rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new));
}
UhVmRpc::ClearHalt(rpc) => {
rpc.handle(async |()| self.partition_unit.clear_halt().await)
Expand Down Expand Up @@ -585,6 +620,12 @@ impl LoadedVm {
}
};

self.teardown().await;

Ok(state)
}

async fn teardown(mut self) {
// Drain resealing before tearing down any storage transport. Removing
// the unit closes its request stream; in-flight I/O finishes first.
if let Some(resealer) = self.hardware_reseal.take() {
Expand All @@ -602,8 +643,6 @@ impl LoadedVm {
if let Some(vmbus) = self.vmbus_server {
vmbus.remove().await.shutdown().await;
}

state
}

/// Handles a servicing request from the host.
Expand Down Expand Up @@ -654,7 +693,9 @@ impl LoadedVm {
.context("failed to notify host of servicing-while-paused failure")?;

if running {
self.start(Some(correlation_id)).await;
self.start(Some(correlation_id))
.await
.context("failed to resume VM after servicing failure")?;
}
false
}
Expand Down Expand Up @@ -884,8 +925,8 @@ impl LoadedVm {
}
}

async fn start(&mut self, correlation_id: Option<Guid>) {
self.state_units.start().await;
async fn start(&mut self, correlation_id: Option<Guid>) -> anyhow::Result<()> {
self.state_units.start().await?;

// Log the boot/blackout time.
let reference_time = ReferenceTime::new(self.partition.reference_time());
Expand All @@ -911,6 +952,8 @@ impl LoadedVm {
"starting VM"
)
}

Ok(())
}

/// Returns true if the VM was previously running.
Expand Down Expand Up @@ -1137,7 +1180,20 @@ impl LoadedVm {
)
.await?;

self.state_units.start_stopped_units().await;
if let Err(error) = self.state_units.start_stopped_units().await {
if let Err(remove_error) = self
.network_settings
.as_mut()
.unwrap()
.remove_network(instance_id)
.await
{
return Err(error.context(format!(
"failed to remove network after start failure: {remove_error:#}"
)));
}
return Err(error);
}
self.emuplat_servicing.netvsp_state.push(save_state);

Ok(())
Expand Down
11 changes: 9 additions & 2 deletions openhcl/underhill_core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -686,11 +686,18 @@ async fn run_control(
continue;
};

let was_resumed = workers
let was_resumed = match workers
.vm_rpc
.call(UhVmRpc::Resume, ())
.await
.context("failed to resumed VM")?;
.context("failed to resume VM")?
{
Ok(was_resumed) => was_resumed,
Err(error) => {
rpc.complete(Err(error));
continue;
}
};

let was_halted = workers
.vm_rpc
Expand Down
1 change: 1 addition & 0 deletions openhcl/underhill_core/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,7 @@ impl Worker for UnderhillVmWorker {
));
tracing::info!(CVM_ALLOWED, "terminating worker");
self.get_thread.join().unwrap();
let state = state?;
if let Some(state) = state {
let params = UnderhillWorkerParameters {
env_cfg: self.env_cfg,
Expand Down
55 changes: 47 additions & 8 deletions openvmm/openvmm_core/src/worker/dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3682,7 +3682,12 @@ impl LoadedVm {
if self.running {
return false;
}
self.state_units.start().await;
// Resume cannot report a failure to its callers, so treat a unit that
// fails to start like any other unrecoverable state unit failure.
self.state_units
.start()
.await
.expect("state units failed to start");
self.running = true;
true
}
Expand Down Expand Up @@ -3724,8 +3729,10 @@ impl LoadedVm {

// Start state units so device config space is accessible. This also
// freezes SMMU capabilities before firmware reads them.
self.state_units.start().await;

self.state_units
.start()
.await
.context("failed to start devices for PCI resource assignment")?;
Comment thread
ppenna marked this conversation as resolved.
Resolved
let result = ecam_config_access::assign_pci_resources_for_root_complexes(
&self.inner.chipset,
&self.inner.pcie_host_bridges,
Expand Down Expand Up @@ -3832,7 +3839,12 @@ impl LoadedVm {
}
Err(err) => {
if stopped {
self.state_units.start().await;
if let Err(start_error) = self.state_units.start().await {
rpc.complete(Err(RemoteError::new(start_error.context(
"worker restart failed and the VM could not resume",
))));
break;
Comment thread
ppenna marked this conversation as resolved.
Resolved
}
}
rpc.complete(Err(RemoteError::new(err)));
}
Expand All @@ -3852,7 +3864,16 @@ impl LoadedVm {
Event::VmRpc(Err(_)) => break,
Event::VmRpc(Ok(message)) => match message {
VmRpc::Reset(rpc) => {
rpc.handle_failable(async |()| self.reset(true).await).await
let result = self.reset(true).await;
// The reset pauses the VM, so state units that still
// run after it fails were left by a failed start, and
// can never start again. Tear the VM down, as after a
// failed restart.
let start_failed = result.is_err() && self.state_units.is_running();
rpc.complete(result.map_err(RemoteError::new));
if start_failed {
break;
}
}
VmRpc::ClearHalt(rpc) => {
rpc.handle(async |()| self.inner.partition_unit.clear_halt().await)
Expand Down Expand Up @@ -3905,8 +3926,12 @@ impl LoadedVm {
resource,
)
.await?;
if let Err(error) = self.state_units.start_stopped_units().await {
// Dropping the channel unit revokes the channel.
drop(device.remove().await);
Comment thread
ppenna marked this conversation as resolved.
Resolved
return Err(error);
}
self.inner.vmbus_devices.push(device);
self.state_units.start_stopped_units().await;
anyhow::Ok(())
})
.await
Expand Down Expand Up @@ -4072,7 +4097,15 @@ impl LoadedVm {
// MSI. The guest may begin probing config space
// immediately after receiving the interrupt, so
// the device must be ready first.
self.state_units.start_stopped_units().await;
if let Err(error) = self.state_units.start_stopped_units().await {
let (_, unit, _) = self
.inner
.pcie_hotplug_devices
.pop()
.expect("device was just pushed");
unit.remove().await;
return Err(error);
}

// Now attach the device and notify the guest.
if let Err(e) = rc.lock().hotplug_add_device(
Expand Down Expand Up @@ -4243,7 +4276,13 @@ impl LoadedVm {
}

device_removal.drain().await;
self.pause().await;
// Stop the state units whenever they run, not only when the VM does: a
// failed start, as in a reset, can leave units running while the VM is
// paused, and a unit whose start is uncertain may not answer its stop.
if self.state_units.is_running() {
self.state_units.stop_for_teardown().await;
}
self.running = false;
self.teardown_chipset_devices().await;
self.inner.partition_unit.teardown().await;
if let Some(vmbus) = self.inner.vmbus_server {
Expand Down
Loading
Loading