Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 95 additions & 45 deletions openhcl/underhill_core/src/dispatch/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc;
#[derive(MeshPayload)]
pub enum UhVmRpc {
Pause(Rpc<(), bool>),
Resume(Rpc<(), bool>),
Resume(FailableRpc<(), bool>),
Save(FailableRpc<(), Vec<u8>>),
ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume
PacketCapture(FailableRpc<PacketCaptureParams<Socket>, PacketCaptureParams<Socket>>),
Expand Down Expand Up @@ -233,9 +233,12 @@ impl LoadedVm {
correlation_id: Option<Guid>,
mut vm_rpc: mesh::Receiver<UhVmRpc>,
mut worker_rpc: mesh::Receiver<WorkerRpc<T>>,
) -> Option<LoadedVmState<T>> {
) -> anyhow::Result<Option<LoadedVmState<T>>> {
if autostart_vps {
self.start(correlation_id).await;
if let Err(error) = self.start(correlation_id).await {
self.teardown().await;
return Err(error.context("failed to autostart VM"));
}
}

// VTL2 settings services
Expand Down Expand Up @@ -323,7 +326,17 @@ impl LoadedVm {
Event::WorkerRpc(message) => match message {
WorkerRpc::Stop => break None,
WorkerRpc::Restart(rpc) => {
let state = async {
enum RestartOutcome<R, S> {
Restart(R, S),
// The restart failed, but the VM is back in its
// previous state, so this worker can keep serving.
Recovered,
// The restart failed and the VM could not be
// resumed, so this worker cannot serve any more.
Unrecoverable,
}

let outcome = async {
let running = self.stop().await;
match self
.save(
Expand All @@ -334,26 +347,40 @@ impl LoadedVm {
)
.await
{
Ok(servicing_state) => Some((rpc, servicing_state)),
Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state),
Err(err) => {
if running {
self.start(None).await;
if let Err(start_error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
start_error.context(format!(
"worker restart failed ({err:#}) and the VM could not resume"
)),
)));
return RestartOutcome::Unrecoverable;
}
}
rpc.complete(Err(RemoteError::new(err)));
None
RestartOutcome::Recovered
}
}
}
.instrument(tracing::info_span!("restart", CVM_ALLOWED))
.await;

if let Some((rpc, servicing_state)) = state {
break Some(LoadedVmState {
restart_rpc: rpc,
servicing_state,
vm_rpc,
control_send: self.control_send.lock().take().unwrap(),
});
match outcome {
RestartOutcome::Restart(rpc, servicing_state) => {
break Some(LoadedVmState {
restart_rpc: rpc,
servicing_state,
vm_rpc,
control_send: self.control_send.lock().take().unwrap(),
});
}
RestartOutcome::Recovered => {}
// The error has already completed the RPC. Tear
// down through the common path rather than keep a
// worker whose state units can no longer run.
RestartOutcome::Unrecoverable => break None,
}
}
WorkerRpc::Inspect(deferred) => deferred.respond(|resp| {
Expand Down Expand Up @@ -403,34 +430,36 @@ impl LoadedVm {
}
Event::UhVmRpc(msg) => match msg {
UhVmRpc::Resume(rpc) => {
rpc.handle(async |()| {
if !self.state_units.is_running() {
self.start(None).await;
true
} else {
false
}
})
.await
if self.state_units.is_running() {
rpc.complete(Ok(false));
} else if let Err(error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
error.context("failed to resume VM"),
)));
} else {
rpc.complete(Ok(true));
}
}
UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await,
UhVmRpc::Save(rpc) => {
rpc.handle_failable(async |()| {
let running = self.stop().await;
let r = self
.save(
SaveReason::Diagnostic,
None,
KeepAliveConfig::Disabled,
KeepAliveConfig::Disabled,
)
.await;
if running {
self.start(None).await;
}
r.map(mesh::payload::encode)
})
.await
let running = self.stop().await;
let result = self
.save(
SaveReason::Diagnostic,
None,
KeepAliveConfig::Disabled,
KeepAliveConfig::Disabled,
)
.await;
if running && let Err(error) = self.start(None).await {
rpc.complete(Err(RemoteError::new(
error.context("failed to resume VM after save"),
)));
// The state units rolled back and cannot run
// again, so tear down rather than keep this worker.
break None;
Comment on lines +458 to +460
}
rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new));
}
UhVmRpc::ClearHalt(rpc) => {
rpc.handle(async |()| self.partition_unit.clear_halt().await)
Expand Down Expand Up @@ -582,6 +611,12 @@ impl LoadedVm {
}
};

self.teardown().await;

Ok(state)
}

async fn teardown(mut self) {
// Drain resealing before tearing down any storage transport. Removing
// the unit closes its request stream; in-flight I/O finishes first.
if let Some(resealer) = self.hardware_reseal.take() {
Expand All @@ -599,8 +634,6 @@ impl LoadedVm {
if let Some(vmbus) = self.vmbus_server {
vmbus.remove().await.shutdown().await;
}

state
}

/// Handles a servicing request from the host.
Expand Down Expand Up @@ -651,7 +684,9 @@ impl LoadedVm {
.context("failed to notify host of servicing-while-paused failure")?;

if running {
self.start(Some(correlation_id)).await;
self.start(Some(correlation_id))
.await
.context("failed to resume VM after servicing failure")?;
}
false
}
Expand Down Expand Up @@ -881,8 +916,8 @@ impl LoadedVm {
}
}

async fn start(&mut self, correlation_id: Option<Guid>) {
self.state_units.start().await;
async fn start(&mut self, correlation_id: Option<Guid>) -> anyhow::Result<()> {
self.state_units.start().await?;

// Log the boot/blackout time.
let reference_time = ReferenceTime::new(self.partition.reference_time());
Expand All @@ -908,6 +943,8 @@ impl LoadedVm {
"starting VM"
)
}

Ok(())
}

/// Returns true if the VM was previously running.
Expand Down Expand Up @@ -1134,7 +1171,20 @@ impl LoadedVm {
)
.await?;

self.state_units.start_stopped_units().await;
if let Err(error) = self.state_units.start_stopped_units().await {
if let Err(remove_error) = self
.network_settings
.as_mut()
.unwrap()
.remove_network(instance_id)
.await
{
return Err(error.context(format!(
"failed to remove network after start failure: {remove_error:#}"
)));
}
return Err(error);
}
self.emuplat_servicing.netvsp_state.push(save_state);

Ok(())
Expand Down
3 changes: 2 additions & 1 deletion openhcl/underhill_core/src/hardware_reseal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ impl inspect::InspectMut for HardwareReseal {
}

impl StateUnit for HardwareReseal {
async fn start(&mut self) {
async fn start(&mut self) -> anyhow::Result<()> {
// Starting or resuming does not create work or reset retry backoff.
// Pending notifications and recovery survive a normal stop/start.
self.schedule.running = true;
Expand All @@ -413,6 +413,7 @@ impl StateUnit for HardwareReseal {
pending_notification = self.notification.pending.load(Ordering::Acquire),
"hardware reseal worker started"
);
Ok(())
}

async fn stop(&mut self) {
Expand Down
13 changes: 10 additions & 3 deletions openhcl/underhill_core/src/hardware_reseal/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,7 @@ async fn finish_attempt(
if start {
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
}
drive_until(run.as_mut(), milestone).await;
Expand Down Expand Up @@ -837,6 +838,7 @@ async fn stopped_worker_keeps_notification_and_start_honors_not_before(driver: D
assert!(notification.pending.load(Ordering::SeqCst));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
assert!(!notification.pending.load(Ordering::SeqCst));
drive_until(run.as_mut(), send.call(StateRequest::Stop, ()))
Expand Down Expand Up @@ -874,6 +876,7 @@ async fn stop_drains_blocked_write_and_final_flush_and_retains_late_event(driver
let mut run = pin!(fixture.worker.run(recv));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
drive_until(run.as_mut(), fixture.io.blocked()).await;
let mut stop = pin!(send.call(StateRequest::Stop, ()));
Expand Down Expand Up @@ -940,6 +943,7 @@ async fn blocked_hardware_keeps_executor_responsive_and_stop_drains_attempt(driv
let mut run = pin!(fixture.worker.run(recv));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
drive_until(run.as_mut(), gate.reached()).await;
{
Expand Down Expand Up @@ -1078,7 +1082,7 @@ async fn failed_report_keeps_recovery_pending_and_retries_successfully(driver: D

let deadline = fixture.worker.schedule.deadline;
let not_before = fixture.worker.schedule.not_before;
fixture.worker.start().await;
fixture.worker.start().await.unwrap();
fixture.worker.stop().await;
assert!(fixture.worker.schedule.force_reseal);
assert_eq!(fixture.worker.schedule.failures, 1);
Expand Down Expand Up @@ -1300,6 +1304,7 @@ async fn changed_report_during_pre_or_post_verification_keeps_recovery_pending(
let mut run = pin!(fixture.worker.run(recv));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
if post_flush {
drive_until(run.as_mut(), fixture.io.blocked()).await;
Expand Down Expand Up @@ -1418,6 +1423,7 @@ async fn migration_during_event_flush_is_detected_without_another_event(driver:
let mut run = pin!(fixture.worker.run(recv));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
drive_until(run.as_mut(), fixture.io.blocked()).await;
fixture.hardware.identity.store(0x73, Ordering::SeqCst);
Expand Down Expand Up @@ -1500,7 +1506,7 @@ async fn restore_and_reset_preserve_upgrade_and_failed_flush_obligation(driver:
assert_eq!(fixture.worker.schedule.deadline, deadline);
assert_eq!(fixture.worker.schedule.not_before, not_before);
assert!(!fixture.worker.notification.pending.load(Ordering::SeqCst));
fixture.worker.start().await;
fixture.worker.start().await.unwrap();
fixture.worker.stop().await;
assert!(fixture.worker.schedule.force_reseal);
assert_eq!(fixture.worker.schedule.failures, 1);
Expand Down Expand Up @@ -1552,6 +1558,7 @@ async fn failed_final_flush_retries_write_even_when_cached_protector_matches(
let mut run = pin!(fixture.worker.run(recv));
drive_until(run.as_mut(), send.call(StateRequest::Start, ()))
.await
.unwrap()
.unwrap();
drive_until(run.as_mut(), fixture.io.blocked()).await;
let stop = send.call(StateRequest::Stop, ());
Expand Down Expand Up @@ -1771,7 +1778,7 @@ async fn running_save_and_restore_fail_without_consuming_pending_work(driver: De
let mut fixture = Fixture::new(&driver, true).await;
fixture.worker.notification.notify();
let saved = fixture.worker.save().await.unwrap().unwrap();
fixture.worker.start().await;
fixture.worker.start().await.unwrap();
let before = worker_snapshot(&fixture.worker);
assert!(fixture.worker.save().await.is_err());
assert!(fixture.worker.restore(saved).await.is_err());
Expand Down
11 changes: 9 additions & 2 deletions openhcl/underhill_core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -686,11 +686,18 @@ async fn run_control(
continue;
};

let was_resumed = workers
let was_resumed = match workers
.vm_rpc
.call(UhVmRpc::Resume, ())
.await
.context("failed to resumed VM")?;
.context("failed to resume VM")?
{
Ok(was_resumed) => was_resumed,
Err(error) => {
rpc.complete(Err(error));
continue;
}
};

let was_halted = workers
.vm_rpc
Expand Down
3 changes: 2 additions & 1 deletion openhcl/underhill_core/src/vmbus_relay_unit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@ impl VmbusRelayHandle {
struct VmbusRelayUnit(HostVmbusTransport);

impl StateUnit for &'_ VmbusRelayUnit {
async fn start(&mut self) {
async fn start(&mut self) -> anyhow::Result<()> {
self.0.start();
Ok(())
}

async fn stop(&mut self) {
Expand Down
1 change: 1 addition & 0 deletions openhcl/underhill_core/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,7 @@ impl Worker for UnderhillVmWorker {
));
tracing::info!(CVM_ALLOWED, "terminating worker");
self.get_thread.join().unwrap();
let state = state?;
if let Some(state) = state {
let params = UnderhillWorkerParameters {
env_cfg: self.env_cfg,
Expand Down
Loading
Loading