Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 98 additions & 45 deletions openhcl/underhill_core/src/dispatch/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc;
#[derive(MeshPayload)]
pub enum UhVmRpc {
Pause(Rpc<(), bool>),
Resume(Rpc<(), bool>),
Resume(FailableRpc<(), bool>),
Save(FailableRpc<(), Vec<u8>>),
ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume
PacketCapture(FailableRpc<PacketCaptureParams<Socket>, PacketCaptureParams<Socket>>),
Expand Down Expand Up @@ -233,9 +233,12 @@ impl LoadedVm {
correlation_id: Option<Guid>,
mut vm_rpc: mesh::Receiver<UhVmRpc>,
mut worker_rpc: mesh::Receiver<WorkerRpc<T>>,
) -> Option<LoadedVmState<T>> {
) -> anyhow::Result<Option<LoadedVmState<T>>> {
if autostart_vps {
self.start(correlation_id).await;
if let Err(error) = self.start(correlation_id).await {
self.teardown().await;
return Err(error.context("failed to autostart VM"));
}
}

// VTL2 settings services
Expand Down Expand Up @@ -323,7 +326,17 @@ impl LoadedVm {
Event::WorkerRpc(message) => match message {
WorkerRpc::Stop => break None,
WorkerRpc::Restart(rpc) => {
let state = async {
enum RestartOutcome<R, S> {
Restart(R, S),
// The restart failed, but the VM is back in its
// previous state, so this worker can keep serving.
Recovered,
// The restart failed and the VM could not be
// resumed, so this worker cannot serve any more.
Unrecoverable,
}

let outcome = async {
let running = self.stop().await;
match self
.save(
Expand All @@ -334,26 +347,40 @@ impl LoadedVm {
)
.await
{
Ok(servicing_state) => Some((rpc, servicing_state)),
Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state),
Err(err) => {
if running {
self.start(None).await;
if let Err(start_error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
start_error.context(format!(
"worker restart failed ({err:#}) and the VM could not resume"
)),
)));
return RestartOutcome::Unrecoverable;
}
}
rpc.complete(Err(RemoteError::new(err)));
None
RestartOutcome::Recovered
}
}
}
.instrument(tracing::info_span!("restart", CVM_ALLOWED))
.await;

if let Some((rpc, servicing_state)) = state {
break Some(LoadedVmState {
restart_rpc: rpc,
servicing_state,
vm_rpc,
control_send: self.control_send.lock().take().unwrap(),
});
match outcome {
RestartOutcome::Restart(rpc, servicing_state) => {
break Some(LoadedVmState {
restart_rpc: rpc,
servicing_state,
vm_rpc,
control_send: self.control_send.lock().take().unwrap(),
});
}
RestartOutcome::Recovered => {}
// The error has already completed the RPC. Tear
// down through the common path rather than keep a
// worker whose state units can no longer run.
RestartOutcome::Unrecoverable => break None,
}
}
WorkerRpc::Inspect(deferred) => deferred.respond(|resp| {
Expand Down Expand Up @@ -403,34 +430,39 @@ impl LoadedVm {
}
Event::UhVmRpc(msg) => match msg {
UhVmRpc::Resume(rpc) => {
rpc.handle(async |()| {
if !self.state_units.is_running() {
self.start(None).await;
true
} else {
false
}
})
.await
if self.state_units.is_running() {
rpc.complete(Ok(false));
} else if let Err(error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
error.context("failed to resume VM"),
)));
// A failed start may leave state units terminal;
// this worker cannot safely continue serving.
break None;
} else {
rpc.complete(Ok(true));
}
}
UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await,
UhVmRpc::Save(rpc) => {
rpc.handle_failable(async |()| {
let running = self.stop().await;
let r = self
.save(
SaveReason::Diagnostic,
None,
KeepAliveConfig::Disabled,
KeepAliveConfig::Disabled,
)
.await;
if running {
self.start(None).await;
}
r.map(mesh::payload::encode)
})
.await
let running = self.stop().await;
let result = self
.save(
SaveReason::Diagnostic,
None,
KeepAliveConfig::Disabled,
KeepAliveConfig::Disabled,
)
.await;
if running && let Err(error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
error.context("failed to resume VM after save"),
)));
// The state units rolled back and cannot run
// again, so tear down rather than keep this worker.
break None;
}
rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new));
}
UhVmRpc::ClearHalt(rpc) => {
rpc.handle(async |()| self.partition_unit.clear_halt().await)
Expand Down Expand Up @@ -582,6 +614,12 @@ impl LoadedVm {
}
};

self.teardown().await;

Ok(state)
}

async fn teardown(mut self) {
// Drain resealing before tearing down any storage transport. Removing
// the unit closes its request stream; in-flight I/O finishes first.
if let Some(resealer) = self.hardware_reseal.take() {
Expand All @@ -599,8 +637,6 @@ impl LoadedVm {
if let Some(vmbus) = self.vmbus_server {
vmbus.remove().await.shutdown().await;
}

state
}

/// Handles a servicing request from the host.
Expand Down Expand Up @@ -651,7 +687,9 @@ impl LoadedVm {
.context("failed to notify host of servicing-while-paused failure")?;

if running {
self.start(Some(correlation_id)).await;
self.start(Some(correlation_id))
.await
.context("failed to resume VM after servicing failure")?;
}
false
}
Expand Down Expand Up @@ -881,8 +919,8 @@ impl LoadedVm {
}
}

async fn start(&mut self, correlation_id: Option<Guid>) {
self.state_units.start().await;
async fn start(&mut self, correlation_id: Option<Guid>) -> anyhow::Result<()> {
self.state_units.start().await?;

// Log the boot/blackout time.
let reference_time = ReferenceTime::new(self.partition.reference_time());
Expand All @@ -908,6 +946,8 @@ impl LoadedVm {
"starting VM"
)
}

Ok(())
}

/// Returns true if the VM was previously running.
Expand Down Expand Up @@ -1134,7 +1174,20 @@ impl LoadedVm {
)
.await?;

self.state_units.start_stopped_units().await;
if let Err(error) = self.state_units.start_stopped_units().await {
if let Err(remove_error) = self
.network_settings
.as_mut()
.unwrap()
.remove_network(instance_id)
.await
{
return Err(error.context(format!(
"failed to remove network after start failure: {remove_error:#}"
)));
}
return Err(error);
}
self.emuplat_servicing.netvsp_state.push(save_state);

Ok(())
Expand Down
11 changes: 9 additions & 2 deletions openhcl/underhill_core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -686,11 +686,18 @@ async fn run_control(
continue;
};

let was_resumed = workers
let was_resumed = match workers
.vm_rpc
.call(UhVmRpc::Resume, ())
.await
.context("failed to resumed VM")?;
.context("failed to resume VM")?
{
Ok(was_resumed) => was_resumed,
Err(error) => {
rpc.complete(Err(error));
continue;
}
};

let was_halted = workers
.vm_rpc
Expand Down
1 change: 1 addition & 0 deletions openhcl/underhill_core/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,7 @@ impl Worker for UnderhillVmWorker {
));
tracing::info!(CVM_ALLOWED, "terminating worker");
self.get_thread.join().unwrap();
let state = state?;
if let Some(state) = state {
let params = UnderhillWorkerParameters {
env_cfg: self.env_cfg,
Expand Down
39 changes: 34 additions & 5 deletions openvmm/openvmm_core/src/worker/dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3697,7 +3697,12 @@ impl LoadedVm {
if self.running {
return false;
}
self.state_units.start().await;
// Resume cannot report a failure to its callers, so treat a unit that
// fails to start like any other unrecoverable state unit failure.
self.state_units
.start()
.await
.expect("state units failed to start");
self.running = true;
true
}
Expand Down Expand Up @@ -3738,7 +3743,10 @@ impl LoadedVm {
let stop_guard = self.inner.partition_unit.temporarily_stop_vps().await;

// Start state units so device config space is accessible.
self.state_units.start().await;
self.state_units
.start()
.await
.context("failed to start devices for PCI resource assignment")?;

let result = ecam_config_access::assign_pci_resources_for_root_complexes(
&self.inner.chipset,
Expand Down Expand Up @@ -3824,7 +3832,12 @@ impl LoadedVm {
}
Err(err) => {
if stopped {
self.state_units.start().await;
if let Err(start_error) = self.state_units.start().await {
rpc.complete(Err(RemoteError::new(start_error.context(
"worker restart failed and the VM could not resume",
))));
break;
}
}
rpc.complete(Err(RemoteError::new(err)));
}
Expand Down Expand Up @@ -3897,7 +3910,15 @@ impl LoadedVm {
)
.await?;
self.inner.vmbus_devices.push(device);
self.state_units.start_stopped_units().await;
if let Err(error) = self.state_units.start_stopped_units().await {
let device = self
.inner
.vmbus_devices
.pop()
.expect("device was just pushed");
drop(device.remove().await);
return Err(error);
}
anyhow::Ok(())
})
.await
Expand Down Expand Up @@ -4043,7 +4064,15 @@ impl LoadedVm {
// MSI. The guest may begin probing config space
// immediately after receiving the interrupt, so
// the device must be ready first.
self.state_units.start_stopped_units().await;
if let Err(error) = self.state_units.start_stopped_units().await {
let (_, unit, _) = self
.inner
.pcie_hotplug_devices
.pop()
.expect("device was just pushed");
unit.remove().await;
return Err(error);
}

// Now attach the device and notify the guest.
if let Err(e) = rc.lock().hotplug_add_device(
Expand Down
10 changes: 7 additions & 3 deletions vm/devices/pci/vpci_relay/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -490,17 +490,21 @@ impl VpciRelay {
.await?
};

entry.insert(RelayedDevice {
let device = RelayedDevice {
bus_instance_id: instance_id,
bus_client: vpci_client,
vpci_device: vpci_device.clone(),
removed,
bus_unit,
device_unit,
ready_to_remove: false,
});
};

state_units.start_stopped_units().await;
if let Err(error) = state_units.start_stopped_units().await {
device.remove().await;
return Err(error);
Comment on lines +503 to +505
}
entry.insert(device);
Ok(())
}
}
Expand Down
6 changes: 5 additions & 1 deletion vmm_core/src/device_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,11 @@ pub async fn build_dynamic_vpci_device(
};

let pending_offer = pending_offer.context("missing deferred VPCI channel offer")?;
state_units.start_stopped_units().await;
if let Err(error) = state_units.start_stopped_units().await {
vpci_unit.remove().await;
pci_unit.remove().await;
Comment on lines +148 to +149
return Err(error);
}
if let Err(error) = pending_offer
.offer_registered(&vpci_bus, driver_source, vmbus, state_units.is_running())
.await
Expand Down
Loading
Loading