Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
54eb2e2
state_unit: make unit start fallible and roll back partial starts
ppenna Sep 25, 2026
18ff657
state_unit: clean up failed starts
ppenna Oct 12, 2026
c598d97
openvmm: preserve teardown after start failures
ppenna Oct 12, 2026
c6499e3
state_unit: stop units whose start fails
ppenna Oct 12, 2026
55942a6
state_unit: preserve cleaned start failures
ppenna Oct 12, 2026
cb87ff9
state_unit: test uncertain start cleanup
ppenna Oct 12, 2026
f408c58
underhill: keep failed diagnostic resume stopped
ppenna Oct 12, 2026
abfb530
state_unit: ignore removed rollback units
ppenna Oct 12, 2026
9bab03f
underhill: forward diagnostic resume errors
ppenna Oct 12, 2026
1ac5d05
state_unit: best-effort stop uncertain starts
ppenna Oct 12, 2026
762b8ea
state_unit: handle rollback transport failures
ppenna Oct 12, 2026
79a67f7
underhill: retain worker after recovery failures
ppenna Oct 12, 2026
de3d100
underhill: make hardware reseal start fallible
ppenna Oct 21, 2026
a52450e
underhill: handle fallible reseal starts in tests
ppenna Oct 23, 2026
b68c328
vmcore: add fallible device start through chipset device workers
ppenna Sep 25, 2026
da9b25c
chipset_device_worker: service memory while starting
ppenna Oct 12, 2026
e31465e
virtio: propagate queue restore start failures
ppenna Oct 12, 2026
4da8697
vmcore: define fallible start rollback contract
ppenna Oct 12, 2026
c7003f7
chipset_device_worker: pin the start request with std::pin::pin
ppenna Oct 13, 2026
4fe5dc9
underhill: tear down the worker after unrecoverable recovery failures
ppenna Nov 2, 2026
960826e
Merge remote-tracking branch 'upstream/main' into feat-state-unit-fal…
ppenna Nov 2, 2026
b21e291
chipset_device_worker: service guest memory while awaiting remote sta…
ppenna Nov 2, 2026
24d13d5
Merge remote-tracking branch 'origin/feat-state-unit-fallible-start' …
ppenna Nov 2, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -653,6 +653,7 @@ dependencies = [
name = "chipset_device_resources"
version = "0.0.0"
dependencies = [
"anyhow",
"async-trait",
"chipset_device",
"guestmem",
Expand Down
140 changes: 95 additions & 45 deletions openhcl/underhill_core/src/dispatch/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc;
#[derive(MeshPayload)]
pub enum UhVmRpc {
Pause(Rpc<(), bool>),
Resume(Rpc<(), bool>),
Resume(FailableRpc<(), bool>),
Save(FailableRpc<(), Vec<u8>>),
ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume
PacketCapture(FailableRpc<PacketCaptureParams<Socket>, PacketCaptureParams<Socket>>),
Expand Down Expand Up @@ -233,9 +233,12 @@ impl LoadedVm {
correlation_id: Option<Guid>,
mut vm_rpc: mesh::Receiver<UhVmRpc>,
mut worker_rpc: mesh::Receiver<WorkerRpc<T>>,
) -> Option<LoadedVmState<T>> {
) -> anyhow::Result<Option<LoadedVmState<T>>> {
if autostart_vps {
self.start(correlation_id).await;
if let Err(error) = self.start(correlation_id).await {
self.teardown().await;
return Err(error.context("failed to autostart VM"));
}
}

// VTL2 settings services
Expand Down Expand Up @@ -323,7 +326,17 @@ impl LoadedVm {
Event::WorkerRpc(message) => match message {
WorkerRpc::Stop => break None,
WorkerRpc::Restart(rpc) => {
let state = async {
enum RestartOutcome<R, S> {
Restart(R, S),
// The restart failed, but the VM is back in its
// previous state, so this worker can keep serving.
Recovered,
// The restart failed and the VM could not be
// resumed, so this worker cannot serve any more.
Unrecoverable,
}

let outcome = async {
let running = self.stop().await;
match self
.save(
Expand All @@ -334,26 +347,40 @@ impl LoadedVm {
)
.await
{
Ok(servicing_state) => Some((rpc, servicing_state)),
Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state),
Err(err) => {
if running {
self.start(None).await;
if let Err(start_error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
start_error.context(format!(
"worker restart failed ({err:#}) and the VM could not resume"
)),
)));
return RestartOutcome::Unrecoverable;
}
}
rpc.complete(Err(RemoteError::new(err)));
None
RestartOutcome::Recovered
}
}
}
.instrument(tracing::info_span!("restart", CVM_ALLOWED))
.await;

if let Some((rpc, servicing_state)) = state {
break Some(LoadedVmState {
restart_rpc: rpc,
servicing_state,
vm_rpc,
control_send: self.control_send.lock().take().unwrap(),
});
match outcome {
RestartOutcome::Restart(rpc, servicing_state) => {
break Some(LoadedVmState {
restart_rpc: rpc,
servicing_state,
vm_rpc,
control_send: self.control_send.lock().take().unwrap(),
});
}
RestartOutcome::Recovered => {}
// The error has already completed the RPC. Tear
// down through the common path rather than keep a
// worker whose state units can no longer run.
RestartOutcome::Unrecoverable => break None,
}
}
WorkerRpc::Inspect(deferred) => deferred.respond(|resp| {
Expand Down Expand Up @@ -403,34 +430,36 @@ impl LoadedVm {
}
Event::UhVmRpc(msg) => match msg {
UhVmRpc::Resume(rpc) => {
rpc.handle(async |()| {
if !self.state_units.is_running() {
self.start(None).await;
true
} else {
false
}
})
.await
if self.state_units.is_running() {
rpc.complete(Ok(false));
} else if let Err(error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
error.context("failed to resume VM"),
)));
} else {
rpc.complete(Ok(true));
}
}
UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await,
UhVmRpc::Save(rpc) => {
rpc.handle_failable(async |()| {
let running = self.stop().await;
let r = self
.save(
SaveReason::Diagnostic,
None,
KeepAliveConfig::Disabled,
KeepAliveConfig::Disabled,
)
.await;
if running {
self.start(None).await;
}
r.map(mesh::payload::encode)
})
.await
let running = self.stop().await;
let result = self
.save(
SaveReason::Diagnostic,
None,
KeepAliveConfig::Disabled,
KeepAliveConfig::Disabled,
)
.await;
if running && let Err(error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
error.context("failed to resume VM after save"),
)));
// The state units rolled back and cannot run
// again, so tear down rather than keep this worker.
break None;
}
rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new));
}
UhVmRpc::ClearHalt(rpc) => {
rpc.handle(async |()| self.partition_unit.clear_halt().await)
Expand Down Expand Up @@ -582,6 +611,12 @@ impl LoadedVm {
}
};

self.teardown().await;

Ok(state)
}

async fn teardown(mut self) {
// Drain resealing before tearing down any storage transport. Removing
// the unit closes its request stream; in-flight I/O finishes first.
if let Some(resealer) = self.hardware_reseal.take() {
Expand All @@ -599,8 +634,6 @@ impl LoadedVm {
if let Some(vmbus) = self.vmbus_server {
vmbus.remove().await.shutdown().await;
}

state
}

/// Handles a servicing request from the host.
Expand Down Expand Up @@ -651,7 +684,9 @@ impl LoadedVm {
.context("failed to notify host of servicing-while-paused failure")?;

if running {
self.start(Some(correlation_id)).await;
self.start(Some(correlation_id))
.await
.context("failed to resume VM after servicing failure")?;
}
false
}
Expand Down Expand Up @@ -881,8 +916,8 @@ impl LoadedVm {
}
}

async fn start(&mut self, correlation_id: Option<Guid>) {
self.state_units.start().await;
async fn start(&mut self, correlation_id: Option<Guid>) -> anyhow::Result<()> {
self.state_units.start().await?;

// Log the boot/blackout time.
let reference_time = ReferenceTime::new(self.partition.reference_time());
Expand All @@ -908,6 +943,8 @@ impl LoadedVm {
"starting VM"
)
}

Ok(())
}

/// Returns true if the VM was previously running.
Expand Down Expand Up @@ -1134,7 +1171,20 @@ impl LoadedVm {
)
.await?;

self.state_units.start_stopped_units().await;
if let Err(error) = self.state_units.start_stopped_units().await {
if let Err(remove_error) = self
.network_settings
.as_mut()
.unwrap()
.remove_network(instance_id)
.await
{
return Err(error.context(format!(
"failed to remove network after start failure: {remove_error:#}"
)));
}
return Err(error);
}
self.emuplat_servicing.netvsp_state.push(save_state);

Ok(())
Expand Down
3 changes: 2 additions & 1 deletion openhcl/underhill_core/src/hardware_reseal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ impl inspect::InspectMut for HardwareReseal {
}

impl StateUnit for HardwareReseal {
async fn start(&mut self) {
async fn start(&mut self) -> anyhow::Result<()> {
// Starting or resuming does not create work or reset retry backoff.
// Pending notifications and recovery survive a normal stop/start.
self.schedule.running = true;
Expand All @@ -413,6 +413,7 @@ impl StateUnit for HardwareReseal {
pending_notification = self.notification.pending.load(Ordering::Acquire),
"hardware reseal worker started"
);
Ok(())
}

async fn stop(&mut self) {
Expand Down
13 changes: 10 additions & 3 deletions openhcl/underhill_core/src/hardware_reseal/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,7 @@ async fn finish_attempt(
if start {
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
}
drive_until(run.as_mut(), milestone).await;
Expand Down Expand Up @@ -837,6 +838,7 @@ async fn stopped_worker_keeps_notification_and_start_honors_not_before(driver: D
assert!(notification.pending.load(Ordering::SeqCst));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
assert!(!notification.pending.load(Ordering::SeqCst));
drive_until(run.as_mut(), send.call(StateRequest::Stop, ()))
Expand Down Expand Up @@ -874,6 +876,7 @@ async fn stop_drains_blocked_write_and_final_flush_and_retains_late_event(driver
let mut run = pin!(fixture.worker.run(recv));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
drive_until(run.as_mut(), fixture.io.blocked()).await;
let mut stop = pin!(send.call(StateRequest::Stop, ()));
Expand Down Expand Up @@ -940,6 +943,7 @@ async fn blocked_hardware_keeps_executor_responsive_and_stop_drains_attempt(driv
let mut run = pin!(fixture.worker.run(recv));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
drive_until(run.as_mut(), gate.reached()).await;
{
Expand Down Expand Up @@ -1078,7 +1082,7 @@ async fn failed_report_keeps_recovery_pending_and_retries_successfully(driver: D

let deadline = fixture.worker.schedule.deadline;
let not_before = fixture.worker.schedule.not_before;
fixture.worker.start().await;
fixture.worker.start().await.unwrap();
fixture.worker.stop().await;
assert!(fixture.worker.schedule.force_reseal);
assert_eq!(fixture.worker.schedule.failures, 1);
Expand Down Expand Up @@ -1300,6 +1304,7 @@ async fn changed_report_during_pre_or_post_verification_keeps_recovery_pending(
let mut run = pin!(fixture.worker.run(recv));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
if post_flush {
drive_until(run.as_mut(), fixture.io.blocked()).await;
Expand Down Expand Up @@ -1418,6 +1423,7 @@ async fn migration_during_event_flush_is_detected_without_another_event(driver:
let mut run = pin!(fixture.worker.run(recv));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
drive_until(run.as_mut(), fixture.io.blocked()).await;
fixture.hardware.identity.store(0x73, Ordering::SeqCst);
Expand Down Expand Up @@ -1500,7 +1506,7 @@ async fn restore_and_reset_preserve_upgrade_and_failed_flush_obligation(driver:
assert_eq!(fixture.worker.schedule.deadline, deadline);
assert_eq!(fixture.worker.schedule.not_before, not_before);
assert!(!fixture.worker.notification.pending.load(Ordering::SeqCst));
fixture.worker.start().await;
fixture.worker.start().await.unwrap();
fixture.worker.stop().await;
assert!(fixture.worker.schedule.force_reseal);
assert_eq!(fixture.worker.schedule.failures, 1);
Expand Down Expand Up @@ -1552,6 +1558,7 @@ async fn failed_final_flush_retries_write_even_when_cached_protector_matches(
let mut run = pin!(fixture.worker.run(recv));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
drive_until(run.as_mut(), fixture.io.blocked()).await;
let stop = send.call(StateRequest::Stop, ());
Expand Down Expand Up @@ -1771,7 +1778,7 @@ async fn running_save_and_restore_fail_without_consuming_pending_work(driver: De
let mut fixture = Fixture::new(&driver, true).await;
fixture.worker.notification.notify();
let saved = fixture.worker.save().await.unwrap().unwrap();
fixture.worker.start().await;
fixture.worker.start().await.unwrap();
let before = worker_snapshot(&fixture.worker);
assert!(fixture.worker.save().await.is_err());
assert!(fixture.worker.restore(saved).await.is_err());
Expand Down
11 changes: 9 additions & 2 deletions openhcl/underhill_core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -686,11 +686,18 @@ async fn run_control(
continue;
};

let was_resumed = workers
let was_resumed = match workers
.vm_rpc
.call(UhVmRpc::Resume, ())
.await
.context("failed to resumed VM")?;
.context("failed to resume VM")?
{
Ok(was_resumed) => was_resumed,
Err(error) => {
rpc.complete(Err(error));
continue;
}
};

let was_halted = workers
.vm_rpc
Expand Down
3 changes: 2 additions & 1 deletion openhcl/underhill_core/src/vmbus_relay_unit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@ impl VmbusRelayHandle {
struct VmbusRelayUnit(HostVmbusTransport);

impl StateUnit for &'_ VmbusRelayUnit {
async fn start(&mut self) {
async fn start(&mut self) -> anyhow::Result<()> {
self.0.start();
Ok(())
}

async fn stop(&mut self) {
Expand Down
1 change: 1 addition & 0 deletions openhcl/underhill_core/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,7 @@ impl Worker for UnderhillVmWorker {
));
tracing::info!(CVM_ALLOWED, "terminating worker");
self.get_thread.join().unwrap();
let state = state?;
if let Some(state) = state {
let params = UnderhillWorkerParameters {
env_cfg: self.env_cfg,
Expand Down
Loading
Loading