Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions crates/openshell-driver-vm/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,11 @@ marked sandboxes without launching compute. Start removes the marker and uses
the normal persisted restore path with the existing overlay. Delete removes the
entire sandbox state directory, including a stop marker and overlay.

After stopping the VM, the driver recovers the overlay's ext4 journal before
removing the previous generation's guest authentication files. If recovery
fails, cleanup returns an error and retains the host generation markers so a
later attempt can recover the disk before changing it.

The host control writes and syncs a terminal tombstone when the canonical main
process exits, before it reports completion and while it retains the boundary
for exec and forwarding. Driver startup reports that sandbox as terminal
Expand Down
40 changes: 40 additions & 0 deletions crates/openshell-driver-vm/src/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6178,6 +6178,7 @@ async fn remove_runtime_generation_material(state_dir: &Path) -> Result<(), Stri
let overlay = sandbox_runtime_disk_paths(state_dir).overlay_disk;
let generation_for_cleanup = generation.clone();
tokio::task::spawn_blocking(move || {
recover_rootfs_image(&overlay)?;
let tls = guest_boundary_tls_paths(&generation_for_cleanup);
for guest_path in [
PathBuf::from(guest_boundary_config_path(&generation_for_cleanup)),
Expand Down Expand Up @@ -8888,6 +8889,45 @@ mod tests {
)
}

#[tokio::test]
async fn generation_cleanup_preserves_markers_when_overlay_recovery_fails() {
let temp = tempfile::tempdir().unwrap();
let state_dir = temp.path();
tokio::fs::write(
state_dir.join(HOST_BOUNDARY_GENERATION_FILE),
b"generation-1\n",
)
.await
.unwrap();
tokio::fs::write(state_dir.join(HOST_AUTH_BUNDLE_FILE), b"auth")
.await
.unwrap();
tokio::fs::write(state_dir.join(HOST_RUNTIME_DESCRIPTOR_FILE), b"descriptor")
.await
.unwrap();
let overlay = sandbox_runtime_disk_paths(state_dir).overlay_disk;
tokio::fs::write(&overlay, b"invalid ext4 image")
.await
.unwrap();

assert!(remove_runtime_generation_material(state_dir).await.is_err());

for marker in [
HOST_BOUNDARY_GENERATION_FILE,
HOST_AUTH_BUNDLE_FILE,
HOST_RUNTIME_DESCRIPTOR_FILE,
] {
assert!(
state_dir.join(marker).is_file(),
"lost {marker} before recovery"
);
}
assert_eq!(
tokio::fs::read(overlay).await.unwrap(),
b"invalid ext4 image"
);
}

#[tokio::test]
async fn explicit_start_clears_stop_and_terminal_markers() {
let temp = tempfile::tempdir().unwrap();
Expand Down
41 changes: 41 additions & 0 deletions e2e/rust/tests/vm_overlay.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,5 +53,46 @@ async fn vm_overlay() {
output.status.code(),
);

for cycle in 0..3 {
let output = openshell_cmd()
.args(["sandbox", "exec", "--name", &sandbox.name, "--no-tty", "--"])
.args(["sh", "-c"])
.arg(format!(
"set -eu; mkdir -p /sandbox/journal-{cycle}; \
for i in $(seq 1 128); do printf '%s\\n' \"$i\" > /sandbox/journal-{cycle}/$i; done; sync"
))
.output()
.await
.expect("write overlay before stop");
assert!(output.status.success(), "overlay writes failed: {output:?}");

for action in ["stop", "start"] {
let output = openshell_cmd()
.args(["sandbox", action, &sandbox.name])
.output()
.await
.expect("run sandbox lifecycle command");
assert!(
output.status.success(),
"{action} failed on cycle {cycle}: {output:?}"
);
}

let output = openshell_cmd()
.args(["sandbox", "exec", "--name", &sandbox.name, "--no-tty", "--"])
.args(["sh", "-c"])
.arg(format!(
"set -eu; test \"$(cat /sandbox/overlay-check)\" = overlay-write; \
for i in $(seq 1 128); do test \"$(cat /sandbox/journal-{cycle}/$i)\" = \"$i\"; done"
))
.output()
.await
.expect("read overlay after restart");
assert!(
output.status.success(),
"overlay contents changed on cycle {cycle}: {output:?}"
);
}

sandbox.cleanup().await;
}
Loading