From e5de0a892813408bf08a36d93ab33b66a9929d18 Mon Sep 17 00:00:00 2001 From: Mike Nguyen Date: Thu, 1 Oct 2026 15:42:48 +0700 Subject: [PATCH 1/2] fix(ssh): persist sandbox host identities Store each sandbox's Ed25519 host key in the gateway credential store and deliver it only to the supervisor. Preserve identity across restarts, delete owned credentials with the sandbox, and expose the public SHA256 fingerprint through sandbox and SSH-session APIs and client SDKs. Cover credential ownership, cancellation, deletion retries, client compatibility, and pinned SSH connections through lifecycle transitions. Closes #3835 Signed-off-by: Mike Nguyen --- Cargo.lock | 1 + crates/openshell-cli/src/run.rs | 1 + crates/openshell-core/src/jwt.rs | 62 +- crates/openshell-driver-docker/src/tests.rs | 1 + crates/openshell-driver-podman/src/driver.rs | 1 + .../openshell-driver-podman/src/isolation.rs | 1 + crates/openshell-driver-vm/src/driver.rs | 1 + crates/openshell-sdk/src/types.rs | 4 + crates/openshell-sdk/tests/client_mock.rs | 24 + crates/openshell-server/Cargo.toml | 1 + .../openshell-server/src/auth/sandbox_jwt.rs | 1 + crates/openshell-server/src/compute/mod.rs | 152 ++++- crates/openshell-server/src/grpc/sandbox.rs | 29 +- crates/openshell-server/src/lib.rs | 3 + .../openshell-server/src/persistence/mod.rs | 13 + .../src/persistence/postgres.rs | 10 +- crates/openshell-server/src/ssh_identity.rs | 599 ++++++++++++++++++ crates/openshell-server/src/storage_proto.rs | 6 +- .../openshell-supervisor-process/Cargo.toml | 2 +- .../src/delegated.rs | 3 + .../openshell-supervisor-process/src/ssh.rs | 146 ++++- .../src/ssh/reconnect_tests.rs | 16 +- crates/openshell-supervisor/src/lib.rs | 8 + docs/how-it-works/sandboxes/overview.mdx | 23 + e2e/rust/tests/ssh_host_identity.rs | 138 ++++ proto/openshell.proto | 7 +- python/openshell/sandbox.py | 3 + python/openshell/sandbox_test.py | 7 + .../v1/internal/converter/sandbox.go | 2 +- .../v1/internal/converter/sandbox_test.go | 2 + sdk/go/openshell/v1/types/sandbox.go | 2 + sdk/go/proto/openshellv1/openshell.pb.go | 19 +- sdk/typescript/src/client.test.ts | 18 + sdk/typescript/src/client.ts | 3 + skills/debug-openshell-cluster/SKILL.md | 1 + skills/openshell-cli/SKILL.md | 7 + 36 files changed, 1261 insertions(+), 56 deletions(-) create mode 100644 crates/openshell-server/src/ssh_identity.rs create mode 100644 e2e/rust/tests/ssh_host_identity.rs diff --git a/Cargo.lock b/Cargo.lock index c6173fee09..2e24a3c7b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4685,6 +4685,7 @@ dependencies = [ "prost-reflect", "prost-types", "protoc-bin-vendored", + "rand 0.10.2", "rand 0.9.4", "rcgen", "reqwest 0.12.28", diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 02c806c421..79b8ceb9c5 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -2793,6 +2793,7 @@ fn sandbox_to_json(sandbox: &Sandbox) -> serde_json::Value { "id": sandbox.object_id(), "name": sandbox.object_name(), "workspace": sandbox.object_workspace(), + "host_key_fingerprint": sandbox.host_key_fingerprint, "labels": labels, "annotations": annotations, "resource_version": meta.map_or(0, |m| m.resource_version), diff --git a/crates/openshell-core/src/jwt.rs b/crates/openshell-core/src/jwt.rs index d1b8d22226..0dac20fce7 100644 --- a/crates/openshell-core/src/jwt.rs +++ b/crates/openshell-core/src/jwt.rs @@ -229,6 +229,46 @@ mod session { } } + /// Private SSH key material is redacted from diagnostics and cleared on drop. + #[derive(Clone)] + pub struct SecretSshHostKey(Zeroizing); + + impl SecretSshHostKey { + #[must_use] + pub fn new(value: String) -> Self { + Self(Zeroizing::new(value)) + } + + #[must_use] + pub fn expose_secret(&self) -> &str { + &self.0 + } + } + + impl fmt::Debug for SecretSshHostKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("SecretSshHostKey([REDACTED])") + } + } + + impl Serialize for SecretSshHostKey { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.expose_secret()) + } + } + + impl<'de> Deserialize<'de> for SecretSshHostKey { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + String::deserialize(deserializer).map(Self::new) + } + } + /// Trusted launch input delivered only to `openshell-supervisor`. #[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -244,6 +284,9 @@ mod session { pub gateway_expires_at: i64, pub sandbox_token: SecretJwt, pub sandbox_expires_at: i64, + /// Never delivered to the workload. Missing only in older bundles. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ssh_host_private_key: Option, } /// Gateway-created authentication input trusted by a compute driver. @@ -329,6 +372,7 @@ mod session { .field("gateway_expires_at", &self.gateway_expires_at) .field("sandbox_token", &"[REDACTED]") .field("sandbox_expires_at", &self.sandbox_expires_at) + .field("ssh_host_private_key", &self.ssh_host_private_key) .finish() } } @@ -1034,6 +1078,7 @@ mod tests { gateway_expires_at: pair.gateway.expires_at, sandbox_token: pair.sandbox.token, sandbox_expires_at: pair.sandbox.expires_at, + ssh_host_private_key: None, }; bundle.validate().expect("non-expiring auth bundle"); } @@ -1060,6 +1105,9 @@ mod tests { gateway_expires_at: pair.gateway.expires_at, sandbox_token: pair.sandbox.token, sandbox_expires_at: pair.sandbox.expires_at, + ssh_host_private_key: Some(SecretSshHostKey::new( + "private-ssh-host-key".to_string(), + )), }; let encoded = serde_json::to_vec(&bundle).expect("serialize auth bundle"); @@ -1078,7 +1126,19 @@ mod tests { let debug = format!("{bundle:?}"); assert!(!debug.contains(bundle.gateway_token.expose_secret())); assert!(!debug.contains(bundle.sandbox_token.expose_secret())); - assert_eq!(debug.matches("[REDACTED]").count(), 2); + assert!(!debug.contains("private-ssh-host-key")); + assert_eq!(debug.matches("[REDACTED]").count(), 3); + assert_eq!( + decoded.ssh_host_private_key.unwrap().expose_secret(), + "private-ssh-host-key" + ); + let mut legacy = serde_json::to_value(&bundle).unwrap(); + legacy + .as_object_mut() + .unwrap() + .remove("ssh_host_private_key"); + let legacy: SupervisorAuthBundle = serde_json::from_value(legacy).unwrap(); + assert!(legacy.ssh_host_private_key.is_none()); } #[derive(Serialize)] diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 8aa05302e0..b2189e43fe 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -38,6 +38,7 @@ fn test_launch_authentication() -> Vec { gateway_expires_at: i64::MAX, sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), sandbox_expires_at: i64::MAX, + ssh_host_private_key: None, }, gateway_id: "gateway-test".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index d3023cddd8..146f263703 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -2062,6 +2062,7 @@ mod tests { gateway_expires_at: i64::MAX, sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), sandbox_expires_at: i64::MAX, + ssh_host_private_key: None, }, gateway_id: "gateway-test".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index ae33796ea6..3eab523746 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -445,6 +445,7 @@ mod tests { gateway_expires_at: i64::MAX, sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), sandbox_expires_at: i64::MAX, + ssh_host_private_key: None, }, gateway_id: "gateway-test".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 3fceaf0e80..1e13e97e35 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -8856,6 +8856,7 @@ mod tests { gateway_expires_at: 1, sandbox_token: SecretJwt::parse(format!("sandbox-{label}")).expect("sandbox token"), sandbox_expires_at: 1, + ssh_host_private_key: None, }, gateway_id: "gateway-a".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-sdk/src/types.rs b/crates/openshell-sdk/src/types.rs index aa2bc71bf9..e4b66942dd 100644 --- a/crates/openshell-sdk/src/types.rs +++ b/crates/openshell-sdk/src/types.rs @@ -384,6 +384,8 @@ pub struct SandboxRef { pub labels: HashMap, pub resource_version: u64, pub exit_code: Option, + /// Public OpenSSH SHA256 host identity; absent on older gateways. + pub host_key_fingerprint: Option, pub created_from_workload_template: Option, /// Service URLs returned by sandbox creation, keyed by service name. The /// empty key identifies the unnamed service. Non-create reads leave this empty. @@ -437,6 +439,8 @@ impl SandboxRef { labels: meta.labels, resource_version: meta.resource_version, exit_code, + host_key_fingerprint: (!sandbox.host_key_fingerprint.is_empty()) + .then_some(sandbox.host_key_fingerprint), created_from_workload_template, service_urls: HashMap::new(), restart_count, diff --git a/crates/openshell-sdk/tests/client_mock.rs b/crates/openshell-sdk/tests/client_mock.rs index 9506160f07..9abc5e9e54 100644 --- a/crates/openshell-sdk/tests/client_mock.rs +++ b/crates/openshell-sdk/tests/client_mock.rs @@ -131,6 +131,11 @@ fn sandbox_with_phase_ws( ..Default::default() }), created_from_workload_template, + host_key_fingerprint: if name == "pinned-identity" { + "SHA256:public-identity".to_string() + } else { + String::new() + }, } } @@ -1291,6 +1296,25 @@ async fn get_sandbox_sends_name_and_maps_phase() { assert_eq!(observed.as_deref(), Some("my-box")); } +#[tokio::test] +async fn get_sandbox_preserves_host_fingerprint_and_accepts_older_gateways() { + let endpoint = start_mock(Arc::new(MockState::default())).await; + let client = connect(&endpoint).await; + let sandbox = client.get_sandbox("pinned-identity").await.unwrap(); + assert_eq!( + sandbox.host_key_fingerprint.as_deref(), + Some("SHA256:public-identity") + ); + assert!( + client + .get_sandbox("legacy") + .await + .unwrap() + .host_key_fingerprint + .is_none() + ); +} + #[tokio::test] async fn get_sandbox_preserves_workload_template_provenance() { let state = Arc::new(MockState::default()); diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 54c44a7bed..dd7b526f23 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -106,6 +106,7 @@ glob = { workspace = true } hex = "0.4" russh = "0.62" rand = { workspace = true } +rand-ssh = { package = "rand", version = "0.10" } petname = "2" ipnet = "2" tempfile = "3" diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index a86eca1fdb..9446512c00 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -232,6 +232,7 @@ impl SandboxSessionJwtAuthority { gateway_expires_at: pair.gateway.expires_at, sandbox_token: pair.sandbox.token, sandbox_expires_at: pair.sandbox.expires_at, + ssh_host_private_key: None, }, gateway_id: self.gateway_id.clone(), verification_keys: self.verification_keys.clone(), diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 264cd77bbb..a08bb31cb2 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -659,6 +659,7 @@ pub struct ComputeRuntime { restart_authority: Arc>>>, restart_notify: Arc, + ssh_identities: Arc>, } pub struct SandboxSyncGuard { @@ -756,9 +757,73 @@ impl ComputeRuntime { rootfs_tar_staging, restart_authority: Arc::new(OnceLock::new()), restart_notify: Arc::new(Notify::new()), + ssh_identities: Arc::new(OnceLock::new()), }) } + pub(crate) fn configure_ssh_identities( + &self, + credentials: crate::credentials::CredentialRuntime, + ) { + let _ = self + .ssh_identities + .set(crate::ssh_identity::SshIdentityStore::new( + self.store.clone(), + credentials, + )); + } + + pub(crate) async fn prepare_ssh_identity( + &self, + sandbox: &mut Sandbox, + authentication: &mut openshell_core::jwt::SandboxLaunchAuthentication, + ) -> Result<(), Status> { + if let Some(identities) = self.ssh_identities.get() { + identities.prepare(sandbox, authentication).await?; + } + Ok(()) + } + + pub(crate) async fn cleanup_ssh_identity(&self, sandbox: &Sandbox) -> Result<(), Status> { + if let Some(identities) = self.ssh_identities.get() { + identities.delete(sandbox).await?; + } + Ok(()) + } + + async fn encode_launch_authentication( + &self, + sandbox: &Sandbox, + mut authentication: openshell_core::jwt::SandboxLaunchAuthentication, + ) -> Result, Status> { + let mut with_identity = sandbox.clone(); + self.prepare_ssh_identity(&mut with_identity, &mut authentication) + .await?; + serialize_launch_authentication(authentication) + } + + async fn encode_persisted_launch_authentication( + &self, + authority: Option<&crate::auth::sandbox_jwt::SandboxSessionJwtAuthority>, + sandbox: &Sandbox, + ) -> Result, Status> { + let Some(authority) = authority else { + return Ok(Vec::new()); + }; + let metadata = sandbox + .metadata + .as_ref() + .ok_or_else(|| Status::failed_precondition("sandbox metadata is missing"))?; + let identity = + crate::auth::sandbox_session::PersistedSandboxIdentity::read(&metadata.annotations) + .map_err(|error| Status::failed_precondition(error.to_string()))?; + self.encode_launch_authentication( + sandbox, + authority.mint_persisted_launch(sandbox.object_id(), &identity)?, + ) + .await + } + /// Serializes sandbox/provider-profile invariant checks and object writes /// across gateway replicas. /// @@ -1098,6 +1163,25 @@ impl ComputeRuntime { if let Some(metadata) = sandbox.metadata.as_mut() { metadata.resource_version = result.resource_version; } + // The parent now owns any staged key, including if this request is + // cancelled. Keep the creation guard until its public identity and + // protected launch bundle have been committed. + let launch_authentication = if let Some(encoded) = launch_authentication { + let authentication = serde_json::from_slice(&encoded) + .map_err(|_| Status::internal("invalid sandbox launch authentication"))?; + let encoded = self + .encode_launch_authentication(&sandbox, authentication) + .await?; + sandbox = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|_| Status::unavailable("reload sandbox SSH identity failed"))? + .ok_or_else(|| Status::aborted("sandbox was removed during creation"))?; + Some(encoded) + } else { + None + }; drop(global_guard); if let Some(token) = sandbox_token @@ -1178,6 +1262,7 @@ impl ComputeRuntime { Ok(sandbox) } Err(status) if status.code() == Code::AlreadyExists => { + self.cleanup_ssh_identity(&sandbox).await?; let _ = self .store .delete(Sandbox::object_type(), sandbox.object_id()) @@ -1186,6 +1271,7 @@ impl ComputeRuntime { Err(Status::already_exists("sandbox already exists")) } Err(status) if status.code() == Code::FailedPrecondition => { + self.cleanup_ssh_identity(&sandbox).await?; let _ = self .store .delete(Sandbox::object_type(), sandbox.object_id()) @@ -1194,6 +1280,7 @@ impl ComputeRuntime { Err(Status::failed_precondition(status.message().to_string())) } Err(err) => { + self.cleanup_ssh_identity(&sandbox).await?; let _ = self .store .delete(Sandbox::object_type(), sandbox.object_id()) @@ -1579,9 +1666,16 @@ impl ComputeRuntime { // Acquiring the lifecycle gate proves that no local worker still // owns this transition. Retry the idempotent driver operation // with the identity committed by the original transition. - let authentication = - serialize_persisted_launch_authentication(authority, ¤t)?; - break (current.clone(), current, authentication); + let authentication = self + .encode_persisted_launch_authentication(authority, ¤t) + .await?; + let refreshed = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|_| Status::unavailable("reload sandbox SSH identity failed"))? + .ok_or_else(|| Status::not_found("sandbox was deleted"))?; + break (refreshed.clone(), refreshed, authentication); } let previous = current.clone(); @@ -1594,11 +1688,14 @@ impl ComputeRuntime { }; let launch_authentication = if let (Some(authority), Some(identity)) = (authority, next_identity.as_ref()) { - serialize_launch_authentication( + self.encode_launch_authentication( + ¤t, authority.mint_persisted_launch(current.object_id(), identity)?, - )? + ) + .await? } else { - serialize_persisted_launch_authentication(authority, ¤t)? + self.encode_persisted_launch_authentication(authority, ¤t) + .await? }; let expected_resource_version = sandbox_resource_version(¤t); let next_identity_for_update = next_identity.clone(); @@ -3774,7 +3871,9 @@ impl ComputeRuntime { return Ok(()); }; - let launch_authentication = serialize_persisted_launch_authentication(authority, &armed) + let launch_authentication = self + .encode_persisted_launch_authentication(authority, &armed) + .await .map_err(|status| status.to_string())?; let generation_id = sandbox_runtime_generation(&armed)?.into_string(); let expected_runtime_identity = sandbox_compute_runtime_identity(¤t); @@ -4811,7 +4910,20 @@ impl ComputeRuntime { // call itself, only on the (instant, non-blocking) decision to // make it. self.spawn_driver_sandbox_cleanup(sandbox.object_id(), sandbox.object_name()); - self.cleanup_sandbox_owned_records(sandbox).await?; + // Fence a delayed identity preparer before reclaiming its key. + // A watch deletion has no request-side Deleting transition. + let deleting = self + .store + .update_message_cas::( + sandbox.object_id(), + sandbox_resource_version(sandbox), + |sandbox| { + sandbox.set_phase(SandboxPhase::Deleting.into()); + }, + ) + .await + .map_err(|error| error.to_string())?; + self.cleanup_sandbox_owned_records(&deleting).await?; } let _ = self @@ -4835,6 +4947,9 @@ impl ComputeRuntime { } async fn cleanup_sandbox_owned_records(&self, sandbox: &Sandbox) -> Result<(), String> { + self.cleanup_ssh_identity(sandbox) + .await + .map_err(|error| error.to_string())?; self.cleanup_sandbox_ssh_sessions(sandbox.object_id(), sandbox.object_workspace()) .await?; self.cleanup_sandbox_service_endpoints(sandbox.object_id(), sandbox.object_workspace()) @@ -6596,25 +6711,6 @@ fn next_runtime_identity( }) } -fn serialize_persisted_launch_authentication( - authority: Option<&crate::auth::sandbox_jwt::SandboxSessionJwtAuthority>, - sandbox: &Sandbox, -) -> Result, Status> { - let Some(authority) = authority else { - return Ok(Vec::new()); - }; - let metadata = sandbox - .metadata - .as_ref() - .ok_or_else(|| Status::failed_precondition("sandbox metadata is missing"))?; - let identity = - crate::auth::sandbox_session::PersistedSandboxIdentity::read(&metadata.annotations) - .map_err(|error| Status::failed_precondition(error.to_string()))?; - serialize_launch_authentication( - authority.mint_persisted_launch(sandbox.object_id(), &identity)?, - ) -} - fn serialize_launch_authentication( authentication: openshell_core::jwt::SandboxLaunchAuthentication, ) -> Result, Status> { @@ -6935,6 +7031,7 @@ pub fn new_test_runtime_with_driver( rootfs_tar_staging: Arc::new(rootfs_tar::RootfsTarStagingRegistry::disabled()), restart_authority: Arc::new(OnceLock::new()), restart_notify: Arc::new(Notify::new()), + ssh_identities: Arc::new(OnceLock::new()), } } @@ -7984,6 +8081,7 @@ mod tests { rootfs_tar_staging: Arc::new(rootfs_tar::RootfsTarStagingRegistry::disabled()), restart_authority: Arc::new(OnceLock::new()), restart_notify: Arc::new(Notify::new()), + ssh_identities: Arc::new(OnceLock::new()), } } diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 074b2590e0..4378b239a8 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -574,6 +574,7 @@ async fn handle_create_sandbox_inner( let now_ms = current_time_ms(); let mut sandbox = Sandbox { + host_key_fingerprint: String::new(), metadata: Some(ObjectMeta { id: id.clone(), name: name.clone(), @@ -1745,7 +1746,7 @@ async fn handle_start_sandbox_inner( })) } -pub fn mint_persisted_authentication( +pub async fn mint_persisted_authentication( state: &ServerState, sandbox: &Sandbox, ) -> Result { @@ -1760,7 +1761,13 @@ pub fn mint_persisted_authentication( let identity = crate::auth::sandbox_session::PersistedSandboxIdentity::read(&metadata.annotations) .map_err(|error| Status::failed_precondition(error.to_string()))?; - authority.mint_persisted_launch(sandbox.object_id(), &identity) + let mut authentication = authority.mint_persisted_launch(sandbox.object_id(), &identity)?; + let mut with_identity = sandbox.clone(); + state + .compute + .prepare_ssh_identity(&mut with_identity, &mut authentication) + .await?; + Ok(authentication) } async fn providers_for_sandbox( @@ -3117,7 +3124,7 @@ pub(super) async fn handle_create_ssh_session( gateway_host, gateway_port: gateway_port.into(), gateway_scheme: scheme.to_string(), - host_key_fingerprint: String::new(), + host_key_fingerprint: sandbox.host_key_fingerprint.clone(), expiration_time: openshell_core::time::optional_timestamp_from_legacy_millis(expires_at_ms) .map_err(|error| Status::internal(error.to_string()))?, })) @@ -8342,11 +8349,9 @@ mod tests { #[tokio::test] async fn concurrent_create_ssh_session_prevents_duplicate_tokens() { let state = test_server_state().await; - state - .store - .put_message(&test_sandbox("work", Vec::new())) - .await - .unwrap(); + let mut sandbox = test_sandbox("work", Vec::new()); + sandbox.host_key_fingerprint = "SHA256:public-host-identity".to_string(); + state.store.put_message(&sandbox).await.unwrap(); // Both requests try to create sessions for the same sandbox // The token generation is random, so we can't force a collision, @@ -8386,8 +8391,12 @@ mod tests { assert!(result1.is_ok(), "first create should succeed"); assert!(result2.is_ok(), "second create should succeed"); - let token1 = result1.unwrap().into_inner().token; - let token2 = result2.unwrap().into_inner().token; + let response1 = result1.unwrap().into_inner(); + let response2 = result2.unwrap().into_inner(); + assert_eq!(response1.host_key_fingerprint, sandbox.host_key_fingerprint); + assert_eq!(response2.host_key_fingerprint, sandbox.host_key_fingerprint); + let token1 = response1.token; + let token2 = response2.token; // Tokens must be different assert_ne!(token1, token2, "tokens should be unique"); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 72efa01fbf..16b0d21632 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -38,6 +38,7 @@ mod readiness; mod sandbox_index; mod sandbox_watch; mod service_routing; +mod ssh_identity; mod ssh_sessions; mod storage_proto; mod supervisor_owner; @@ -414,6 +415,7 @@ impl ServerState { credentials: credentials::CredentialRuntime, ) -> Self { let replica_id = compute::lease::replica_id(); + compute.configure_ssh_identities(credentials.clone()); let peer_endpoint = derive_peer_endpoint(&config); let grpc_rate_limiter = multiplex::GrpcRateLimiter::from_config(&config); let admin_role = config @@ -970,6 +972,7 @@ pub(crate) async fn run_server( return Ok(Vec::new()); } let authentication = grpc::mint_persisted_authentication(&state, &sandbox) + .await .map_err(|error| error.to_string())?; serde_json::to_vec(&authentication) .map_err(|error| format!("encode launch authentication: {error}")) diff --git a/crates/openshell-server/src/persistence/mod.rs b/crates/openshell-server/src/persistence/mod.rs index 2513e80c10..1721cf7931 100644 --- a/crates/openshell-server/src/persistence/mod.rs +++ b/crates/openshell-server/src/persistence/mod.rs @@ -299,6 +299,19 @@ impl Store { } } + /// Independent of the cross-object lock: creation already holds that + /// lock when it provisions a supervisor's durable SSH identity. + pub(crate) async fn acquire_ssh_identity_mutation_guard( + &self, + ) -> PersistenceResult { + match self { + Self::Postgres(store) => Ok(DistributedMutationGuard { + _postgres: Some(store.acquire_mutation_lock(0x4f53_5348_484f_5354).await?), + }), + Self::Sqlite(_) => Ok(DistributedMutationGuard { _postgres: None }), + } + } + /// Connect to a persistence store based on the database URL. pub async fn connect(url: &str) -> CoreResult { if url.starts_with("postgres://") || url.starts_with("postgresql://") { diff --git a/crates/openshell-server/src/persistence/postgres.rs b/crates/openshell-server/src/persistence/postgres.rs index 14bae6dfb1..22a86f9f3e 100644 --- a/crates/openshell-server/src/persistence/postgres.rs +++ b/crates/openshell-server/src/persistence/postgres.rs @@ -116,6 +116,14 @@ impl PostgresStore { pub(super) async fn acquire_cross_object_lock( &self, + ) -> PersistenceResult { + self.acquire_mutation_lock(CROSS_OBJECT_ADVISORY_LOCK_KEY) + .await + } + + pub(super) async fn acquire_mutation_lock( + &self, + key: i64, ) -> PersistenceResult { let mut connection = self.pool.acquire().await.map_err(|e| map_db_error(&e))?; connection.close_on_drop(); @@ -125,7 +133,7 @@ impl PostgresStore { .await .map_err(|e| map_db_error(&e))?; sqlx::query("SELECT pg_advisory_lock($1)") - .bind(CROSS_OBJECT_ADVISORY_LOCK_KEY) + .bind(key) .execute(&mut *connection) .await .map_err(|e| map_db_error(&e))?; diff --git a/crates/openshell-server/src/ssh_identity.rs b/crates/openshell-server/src/ssh_identity.rs new file mode 100644 index 0000000000..74e43c9ffd --- /dev/null +++ b/crates/openshell-server/src/ssh_identity.rs @@ -0,0 +1,599 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox-lifetime SSH identities. Public objects contain only fingerprints; +//! the credential driver owns private keys and this internal row owns handles. + +use std::collections::HashMap; +use std::sync::Arc; + +use openshell_core::jwt::{SandboxLaunchAuthentication, SecretSshHostKey}; +use openshell_core::proto::Sandbox; +use openshell_core::proto::datamodel::v1::{CredentialHandle, ObjectMeta, Provider}; +use openshell_core::{ObjectId as _, ObjectName as _, ObjectWorkspace as _}; +use prost::Message as _; +use russh::keys::{Algorithm, HashAlg, PrivateKey}; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; +use tonic::Status; + +use crate::credentials::CredentialRuntime; +use crate::persistence::{PersistenceError, Store}; + +const OBJECT_TYPE: &str = "sandbox_ssh_identity_v1"; +const CANDIDATE_TYPE: &str = "sandbox_ssh_identity_candidate_v1"; +const CREDENTIAL_KEY: &str = "SSH_HOST_PRIVATE_KEY"; + +#[derive(Clone)] +pub struct SshIdentityStore { + store: Arc, + credentials: CredentialRuntime, + mutation: Arc>, +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::jwt::{CredentialEpoch, SecretJwt, SessionRotation, SupervisorAuthBundle}; + use openshell_core::{SandboxSessionId, sandbox_generation::SandboxGenerationId}; + + async fn fixture() -> SshIdentityStore { + let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); + let config = crate::Config::new(None) + .with_credential_drivers(["test-static"]) + .with_default_credential_driver(Some("test-static")); + let credentials = + CredentialRuntime::from_config_with_store(&config, store.clone()).unwrap(); + store.put_message(&sandbox("sandbox-a")).await.unwrap(); + SshIdentityStore::new(store, credentials) + } + + fn sandbox(id: &str) -> Sandbox { + Sandbox { + metadata: Some(ObjectMeta { + id: id.to_string(), + name: "same-name".to_string(), + workspace: "default".to_string(), + ..Default::default() + }), + ..Default::default() + } + } + + fn authentication() -> SandboxLaunchAuthentication { + SandboxLaunchAuthentication { + supervisor: SupervisorAuthBundle { + session_id: SandboxSessionId::new(), + runtime_generation: SandboxGenerationId::parse("generation").unwrap(), + session_rotation: SessionRotation::new(1).unwrap(), + auth_epoch: CredentialEpoch::new(1).unwrap(), + gateway_token: SecretJwt::parse("gateway-token").unwrap(), + gateway_expires_at: 0, + sandbox_token: SecretJwt::parse("sandbox-token").unwrap(), + sandbox_expires_at: 0, + ssh_host_private_key: None, + }, + gateway_id: "gateway".to_string(), + verification_keys: Vec::new(), + } + } + + #[tokio::test] + async fn concurrent_candidates_keep_one_resolvable_identity() { + let identities = fixture().await; + let sb = sandbox("sandbox-a"); + let (entered, release) = identities.credentials.gate_next_store(); + let first = identities.clone(); + let sb1 = sb.clone(); + let task = tokio::spawn(async move { first.get_or_create(&sb1).await.unwrap() }); + entered.await.unwrap(); + let winner = identities.get_or_create(&sb).await.unwrap(); + release.send(()).unwrap(); + let loser = task.await.unwrap(); + assert_eq!(winner.fingerprint, loser.fingerprint); + assert_eq!(identities.credentials.stored_credential_count(), Some(1)); + identities + .prepare(&mut sb.clone(), &mut authentication()) + .await + .unwrap(); + } + + #[tokio::test] + async fn identity_survives_reload_and_is_never_public() { + let identities = fixture().await; + let mut sb = sandbox("sandbox-a"); + let mut first = authentication(); + identities.prepare(&mut sb, &mut first).await.unwrap(); + let restored = + SshIdentityStore::new(identities.store.clone(), identities.credentials.clone()); + let mut next = authentication(); + restored.prepare(&mut sb, &mut next).await.unwrap(); + let private = first.supervisor.ssh_host_private_key.unwrap(); + assert_eq!( + private.expose_secret(), + next.supervisor + .ssh_host_private_key + .as_ref() + .unwrap() + .expose_secret() + ); + let key = PrivateKey::from_openssh(private.expose_secret()).unwrap(); + assert_eq!( + sb.host_key_fingerprint, + key.public_key().fingerprint(HashAlg::Sha256).to_string() + ); + assert!(!format!("{sb:?}").contains("PRIVATE KEY")); + assert!(!format!("{next:?}").contains("PRIVATE KEY")); + let stored = identities + .store + .get(OBJECT_TYPE, &SshIdentityStore::identity_id(&sb)) + .await + .unwrap() + .unwrap(); + assert!(!String::from_utf8_lossy(&stored.payload).contains("PRIVATE KEY")); + } + + #[tokio::test] + async fn deletion_retries_before_releasing_key_ownership() { + let identities = fixture().await; + let sb = sandbox("sandbox-a"); + let original = identities.get_or_create(&sb).await.unwrap(); + identities.credentials.fail_next_delete(); + assert!(identities.delete(&sb).await.is_err()); + assert_eq!( + identities.load(&sb).await.unwrap().unwrap().fingerprint, + original.fingerprint + ); + identities.delete(&sb).await.unwrap(); + assert!(identities.load(&sb).await.unwrap().is_none()); + assert_eq!(identities.credentials.stored_credential_count(), Some(0)); + identities.delete(&sb).await.unwrap(); + let replacement = identities + .get_or_create(&sandbox("sandbox-b")) + .await + .unwrap(); + assert_ne!(original.fingerprint, replacement.fingerprint); + } + + #[tokio::test] + async fn cancelled_prepare_finishes_before_deletion_and_cannot_recreate_identity() { + let identities = fixture().await; + let (entered, release) = identities.credentials.gate_next_store(); + let preparing = identities.clone(); + let task = tokio::spawn(async move { + preparing + .prepare(&mut sandbox("sandbox-a"), &mut authentication()) + .await + }); + entered.await.unwrap(); + task.abort(); + let deleting = identities + .store + .update_message_cas::("sandbox-a", 0, |sb| { + sb.set_phase(openshell_core::proto::SandboxPhase::Deleting.into()); + }) + .await + .unwrap(); + let cleanup = identities.clone(); + let deleted = tokio::spawn(async move { cleanup.delete(&deleting).await }); + release.send(()).unwrap(); + deleted.await.unwrap().unwrap(); + assert_eq!(identities.credentials.stored_credential_count(), Some(0)); + assert!( + identities + .prepare(&mut sandbox("sandbox-a"), &mut authentication()) + .await + .is_err() + ); + identities + .store + .delete("sandbox", "sandbox-a") + .await + .unwrap(); + assert!( + identities + .prepare(&mut sandbox("sandbox-a"), &mut authentication()) + .await + .is_err() + ); + assert_eq!(identities.credentials.stored_credential_count(), Some(0)); + } + + #[tokio::test] + async fn losing_candidate_cleanup_failure_remains_owned_for_deletion() { + let identities = fixture().await; + let sb = sandbox("sandbox-a"); + let (entered, release) = identities.credentials.gate_next_store(); + let candidate = identities.clone(); + let task = + tokio::spawn(async move { candidate.get_or_create(&sandbox("sandbox-a")).await }); + entered.await.unwrap(); + identities.get_or_create(&sb).await.unwrap(); + identities.credentials.fail_next_delete(); + release.send(()).unwrap(); + assert!(task.await.unwrap().is_err()); + assert_eq!(identities.credentials.stored_credential_count(), Some(2)); + identities.delete(&sb).await.unwrap(); + assert_eq!(identities.credentials.stored_credential_count(), Some(0)); + assert!( + identities + .store + .list_by_scope(CANDIDATE_TYPE, "sandbox-a", 100, 0) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn published_identity_is_never_replaced_when_storage_is_inconsistent() { + let identities = fixture().await; + let mut sb = sandbox("sandbox-a"); + identities + .prepare(&mut sb, &mut authentication()) + .await + .unwrap(); + identities + .store + .update_message_cas::("sandbox-a", 0, |sb| { + sb.host_key_fingerprint = "SHA256:inconsistent".to_string(); + }) + .await + .unwrap(); + assert!( + identities + .prepare(&mut sb, &mut authentication()) + .await + .is_err() + ); + let stored = identities + .store + .get_message::("sandbox-a") + .await + .unwrap() + .unwrap(); + assert_eq!(stored.host_key_fingerprint, "SHA256:inconsistent"); + identities + .store + .delete(OBJECT_TYPE, &SshIdentityStore::identity_id(&sb)) + .await + .unwrap(); + assert!( + identities + .prepare(&mut sb, &mut authentication()) + .await + .is_err() + ); + assert_eq!(identities.credentials.stored_credential_count(), Some(1)); + } + + #[tokio::test] + async fn default_credential_store_persists_key_across_runtime_reconstruction() { + let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); + let config = crate::Config::new(None); + let first = SshIdentityStore::new( + store.clone(), + CredentialRuntime::from_config_with_store(&config, store.clone()).unwrap(), + ); + let mut sb = sandbox("sandbox-a"); + first.store.put_message(&sb).await.unwrap(); + first.prepare(&mut sb, &mut authentication()).await.unwrap(); + let fingerprint = sb.host_key_fingerprint.clone(); + let next = SshIdentityStore::new( + store.clone(), + CredentialRuntime::from_config_with_store(&config, store).unwrap(), + ); + next.prepare(&mut sb, &mut authentication()).await.unwrap(); + assert_eq!(fingerprint, sb.host_key_fingerprint); + next.delete(&sb).await.unwrap(); + } +} + +#[derive(Serialize, Deserialize)] +struct StoredIdentity { + fingerprint: String, + // Protobuf bytes avoid introducing another durable handle schema. + credential_handle: Vec, +} + +impl SshIdentityStore { + pub(crate) fn new(store: Arc, credentials: CredentialRuntime) -> Self { + Self { + store, + credentials, + mutation: Arc::new(Mutex::new(())), + } + } + + /// Keep credential staging and publication alive if the calling RPC is + /// cancelled. The independent identity lock fences parent deletion on + /// every replica without nesting the creation cross-object lock. + pub(crate) async fn prepare( + &self, + sandbox: &mut Sandbox, + authentication: &mut SandboxLaunchAuthentication, + ) -> Result<(), Status> { + let identities = self.clone(); + let id = sandbox.object_id().to_string(); + let (updated, private_key) = tokio::spawn(async move { + let _local = identities.mutation.clone().lock_owned().await; + let _distributed = identities + .store + .acquire_ssh_identity_mutation_guard() + .await + .map_err(|_| Status::unavailable("lock sandbox SSH identity failed"))?; + let mut current = identities + .store + .get_message::(&id) + .await + .map_err(|_| Status::unavailable("load sandbox SSH identity owner failed"))? + .ok_or_else(|| Status::not_found("sandbox SSH identity owner was deleted"))?; + if current.phase() == i32::from(openshell_core::proto::SandboxPhase::Deleting) { + return Err(Status::failed_precondition( + "sandbox SSH identity owner is deleting", + )); + } + let private_key = identities.prepare_locked(&mut current).await?; + Ok::<_, Status>((current, private_key)) + }) + .await + .map_err(|_| Status::internal("sandbox SSH identity worker failed"))??; + *sandbox = updated; + authentication.supervisor.ssh_host_private_key = Some(private_key); + Ok(()) + } + + async fn prepare_locked(&self, sandbox: &mut Sandbox) -> Result { + let identity = self.get_or_create(sandbox).await?; + if !sandbox.host_key_fingerprint.is_empty() + && sandbox.host_key_fingerprint != identity.fingerprint + { + return Err(Status::internal( + "sandbox SSH identity does not match its published fingerprint", + )); + } + let handle = CredentialHandle::decode(identity.credential_handle.as_slice()) + .map_err(|_| Status::internal("invalid SSH identity credential handle"))?; + let provider = Self::owner( + sandbox, + HashMap::from([(CREDENTIAL_KEY.to_string(), handle)]), + ); + let mut resolved = self + .credentials + .resolve_provider_handles(&provider, 0) + .await?; + let private_key = resolved + .values + .remove(CREDENTIAL_KEY) + .ok_or_else(|| Status::unavailable("sandbox SSH host key is unavailable"))?; + let key = PrivateKey::from_openssh(&private_key) + .map_err(|_| Status::internal("invalid stored sandbox SSH host key"))?; + if key.public_key().fingerprint(HashAlg::Sha256).to_string() != identity.fingerprint { + return Err(Status::internal( + "sandbox SSH identity does not match its stored key", + )); + } + if sandbox.host_key_fingerprint.is_empty() { + let fingerprint = identity.fingerprint; + *sandbox = self + .store + .update_message_cas::(sandbox.object_id(), 0, |sandbox| { + sandbox.host_key_fingerprint.clone_from(&fingerprint); + }) + .await + .map_err(|_| Status::aborted("publish sandbox SSH identity failed"))?; + } + Ok(SecretSshHostKey::new(private_key)) + } + + fn identity_id(sandbox: &Sandbox) -> String { + format!("sandbox-ssh:{}", sandbox.object_id()) + } + + fn owner(sandbox: &Sandbox, handles: HashMap) -> Provider { + Provider { + metadata: Some(ObjectMeta { + id: sandbox.object_id().to_string(), + name: format!("sandbox-ssh-{}", sandbox.object_id()), + workspace: sandbox.object_workspace().to_string(), + ..Default::default() + }), + credential_handles: handles, + ..Default::default() + } + } + + async fn load(&self, sandbox: &Sandbox) -> Result, Status> { + self.store + .get(OBJECT_TYPE, &Self::identity_id(sandbox)) + .await + .map_err(|_| Status::unavailable("load sandbox SSH identity failed"))? + .map(|record| { + serde_json::from_slice(&record.payload) + .map_err(|_| Status::internal("invalid stored sandbox SSH identity")) + }) + .transpose() + } + + async fn get_or_create(&self, sandbox: &Sandbox) -> Result { + if let Some(identity) = self.load(sandbox).await? { + return Ok(identity); + } + if !sandbox.host_key_fingerprint.is_empty() { + return Err(Status::unavailable( + "sandbox SSH host key is missing; refusing to replace its identity", + )); + } + let key = PrivateKey::random(&mut rand_ssh::rng(), Algorithm::Ed25519) + .map_err(|_| Status::internal("generate sandbox SSH host key failed"))?; + let private_key = key + .to_openssh(russh::keys::ssh_key::LineEnding::default()) + .map_err(|_| Status::internal("encode sandbox SSH host key failed"))?; + let owner = Self::owner(sandbox, HashMap::new()); + // A distinct candidate path is essential for K8s/Vault: losing a CAS + // must never overwrite or delete the winning candidate's credential. + let candidate_id = uuid::Uuid::new_v4().to_string(); + let handles = self + .credentials + .store_provider_credentials_with_object_id( + owner.object_name(), + sandbox.object_workspace(), + sandbox.object_id(), + &candidate_id, + &HashMap::from([(CREDENTIAL_KEY.to_string(), private_key.to_string())]), + &HashMap::new(), + ) + .await?; + let handle = handles + .get(CREDENTIAL_KEY) + .ok_or_else(|| Status::internal("credential driver omitted sandbox SSH host key"))?; + let identity = StoredIdentity { + fingerprint: key.public_key().fingerprint(HashAlg::Sha256).to_string(), + credential_handle: handle.encode_to_vec(), + }; + let payload = serde_json::to_vec(&identity) + .map_err(|_| Status::internal("encode sandbox SSH identity failed"))?; + // Retain losing handles until cleanup succeeds, so deletion can retry + // a credential-driver failure instead of losing ownership of the key. + if self + .store + .create_scoped( + CANDIDATE_TYPE, + &candidate_id, + &candidate_id, + sandbox.object_workspace(), + sandbox.object_id(), + &payload, + None, + ) + .await + .is_err() + { + self.credentials + .delete_provider_credential_handles( + owner.object_name(), + sandbox.object_workspace(), + sandbox.object_id(), + &handles, + ) + .await?; + return Err(Status::unavailable("stage sandbox SSH identity failed")); + } + match self + .store + .create_scoped( + OBJECT_TYPE, + &Self::identity_id(sandbox), + sandbox.object_id(), + sandbox.object_workspace(), + sandbox.object_id(), + &payload, + None, + ) + .await + { + Ok(_) => { + self.store + .delete(CANDIDATE_TYPE, &candidate_id) + .await + .map_err(|_| { + Status::unavailable("finish sandbox SSH identity staging failed") + })?; + Ok(identity) + } + Err(error) => { + self.credentials + .delete_provider_credential_handles( + owner.object_name(), + sandbox.object_workspace(), + sandbox.object_id(), + &handles, + ) + .await?; + self.store + .delete(CANDIDATE_TYPE, &candidate_id) + .await + .map_err(|_| { + Status::unavailable("finish sandbox SSH identity cleanup failed") + })?; + if matches!(error, PersistenceError::UniqueViolation { .. }) { + self.load(sandbox).await?.ok_or_else(|| { + Status::aborted("sandbox SSH identity was concurrently removed") + }) + } else { + Err(Status::unavailable("persist sandbox SSH identity failed")) + } + } + } + } + + /// Credential deletion precedes row deletion so failure retains ownership + /// and can be retried by the normal sandbox deletion controller. + pub(crate) async fn delete(&self, sandbox: &Sandbox) -> Result<(), Status> { + let identities = self.clone(); + let sandbox = sandbox.clone(); + tokio::spawn(async move { + let _local = identities.mutation.clone().lock_owned().await; + let _distributed = identities + .store + .acquire_ssh_identity_mutation_guard() + .await + .map_err(|_| Status::unavailable("lock sandbox SSH identity failed"))?; + identities.delete_locked(&sandbox).await + }) + .await + .map_err(|_| Status::internal("sandbox SSH identity deletion worker failed"))? + } + + async fn delete_locked(&self, sandbox: &Sandbox) -> Result<(), Status> { + loop { + let candidates = self + .store + .list_by_scope(CANDIDATE_TYPE, sandbox.object_id(), 100, 0) + .await + .map_err(|_| Status::unavailable("load staged sandbox SSH identities failed"))?; + if candidates.is_empty() { + break; + } + for candidate in candidates { + let identity: StoredIdentity = serde_json::from_slice(&candidate.payload) + .map_err(|_| Status::internal("invalid staged sandbox SSH identity"))?; + self.delete_credential(sandbox, &identity).await?; + self.store + .delete(CANDIDATE_TYPE, &candidate.id) + .await + .map_err(|_| { + Status::unavailable("delete staged sandbox SSH identity failed") + })?; + } + } + let Some(identity) = self.load(sandbox).await? else { + return Ok(()); + }; + self.delete_credential(sandbox, &identity).await?; + self.store + .delete(OBJECT_TYPE, &Self::identity_id(sandbox)) + .await + .map_err(|_| Status::unavailable("delete sandbox SSH identity failed"))?; + Ok(()) + } + + async fn delete_credential( + &self, + sandbox: &Sandbox, + identity: &StoredIdentity, + ) -> Result<(), Status> { + let handle = CredentialHandle::decode(identity.credential_handle.as_slice()) + .map_err(|_| Status::internal("invalid SSH identity credential handle"))?; + let owner = Self::owner(sandbox, HashMap::new()); + self.credentials + .delete_provider_credential_handles( + owner.object_name(), + sandbox.object_workspace(), + sandbox.object_id(), + &HashMap::from([(CREDENTIAL_KEY.to_string(), handle)]), + ) + .await?; + Ok(()) + } +} diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index e6d8730aa0..36a12ab281 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -128,10 +128,12 @@ mod tests { // the additive file map while retaining the rest of the response. // Service authorization also extends both schemas additively. Legacy // payloads retain the safe Strip default. + // Sandbox field 6 adds its public SSH fingerprint. Older persisted + // sandboxes decode with an empty fingerprint, provisioned at next launch. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "2e156c6ad3c8eb51bcd30dc13b173fe339b38207a1b1f98f7be2e0cad8e3bd45"; + "354adc0a5945a9105b44602923bfdb8405766e91291fc651da5a9af968e43987"; const DURABLE_SCHEMA_SHA256: &str = - "38165d9d76f49fcfe98a12f241e032838a2376c1d1a87ea2796fd33b9b1a3541"; + "c2db375dc0b0ec51e6ef6ad6715704f3bec9cd11ccb05ced254a2c4efcbabe73"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = "761dea31a521b0650840fe2a823ad6e36a265ed323ba4506889781d630df0ee3"; // A persisted Sandbox without endpoint status retains its lifecycle fields; diff --git a/crates/openshell-supervisor-process/Cargo.toml b/crates/openshell-supervisor-process/Cargo.toml index 39d48a96e5..e8baa5a4b1 100644 --- a/crates/openshell-supervisor-process/Cargo.toml +++ b/crates/openshell-supervisor-process/Cargo.toml @@ -11,7 +11,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -openshell-core = { path = "../openshell-core" } +openshell-core = { path = "../openshell-core", features = ["jwt"] } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 87101f62a2..3bc4638f07 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -87,6 +87,7 @@ pub async fn start_boundary_access( port_forward: Arc, agent: Arc, supervisor_session_updates: Option>>, + host_key: Option, ) -> Result { let instance_id = uuid::Uuid::new_v4().to_string(); let terminating = Arc::new(AtomicBool::new(false)); @@ -106,6 +107,7 @@ pub async fn start_boundary_access( .await .map_err(|error| miette::miette!(error.to_string()))?; let main_session = crate::main_session::MainSession::from_boundary(attachment, agent); + let host_key = host_key.ok_or_else(|| miette::miette!("sandbox SSH host key is missing"))?; let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); let listen_path = ssh_socket_path.clone(); @@ -120,6 +122,7 @@ pub async fn start_boundary_access( ssh_port_forward, boundary_exec, Some(ssh_main_session), + host_key, ) .await { diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 40e0db1668..d3b95e9721 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -67,14 +67,27 @@ type SshServerInit = ( Option>, ); +/// A managed SSH server must never silently replace its pinned identity. +pub fn parse_host_key(key: Option<&openshell_core::jwt::SecretSshHostKey>) -> Result { + let key = key.ok_or_else(|| { + miette::miette!("sandbox SSH host key is missing; restart with a matching gateway release") + })?; + let key = PrivateKey::from_openssh(key.expose_secret()) + .map_err(|_| miette::miette!("invalid sandbox SSH host key"))?; + if key.algorithm() != Algorithm::Ed25519 || key.is_encrypted() { + return Err(miette::miette!( + "sandbox SSH host key must be an unencrypted Ed25519 key" + )); + } + Ok(key) +} + fn ssh_server_init( listen_path: &Path, ca_file_paths: &Option<(PathBuf, PathBuf)>, shared_socket: bool, + host_key: PrivateKey, ) -> Result { - let mut rng = rand::rng(); - let host_key = PrivateKey::random(&mut rng, Algorithm::Ed25519).into_diagnostic()?; - let mut config = russh::server::Config { server_id: russh::SshId::Standard(Cow::Owned(format!("SSH-2.0-OpenShell_{VERSION}"))), auth_rejection_time: Duration::from_secs(1), @@ -140,9 +153,10 @@ pub async fn run_ssh_server( port_forward: Arc, boundary_exec: Arc, main_session: Option>, + host_key: PrivateKey, ) -> Result<()> { let (listener, config, _ca_paths) = - match ssh_server_init(&listen_path, &ca_file_paths, shared_socket) { + match ssh_server_init(&listen_path, &ca_file_paths, shared_socket, host_key) { Ok(v) => { // Signal that the SSH server has bound the socket and is ready to // accept connections. The parent task awaits this before spawning @@ -1274,6 +1288,106 @@ mod tests { pub(super) struct AcceptAnyServerKey; + struct PinnedServerKey(String); + + impl russh::client::Handler for PinnedServerKey { + type Error = russh::Error; + + async fn check_server_key( + &mut self, + key: &russh::keys::PublicKey, + ) -> Result { + Ok(key.fingerprint(russh::keys::HashAlg::Sha256).to_string() == self.0) + } + } + + #[test] + fn managed_host_key_rejects_missing_and_invalid_material() { + assert!(parse_host_key(None).is_err()); + let secret = openshell_core::jwt::SecretSshHostKey::new("secret-invalid-key".to_string()); + let error = parse_host_key(Some(&secret)).unwrap_err().to_string(); + assert!(!error.contains(secret.expose_secret())); + let key = PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(); + let secret = openshell_core::jwt::SecretSshHostKey::new( + key.to_openssh(russh::keys::ssh_key::LineEnding::default()) + .unwrap() + .to_string(), + ); + let decoded = parse_host_key(Some(&secret)).unwrap(); + assert_eq!(key.public_key(), decoded.public_key()); + } + + #[cfg(unix)] + #[tokio::test] + async fn pinned_identity_matches_direct_and_relayed_ssh_after_listener_restart() { + let key = PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(); + let fingerprint = key + .public_key() + .fingerprint(russh::keys::HashAlg::Sha256) + .to_string(); + let temp = tempfile::tempdir().unwrap(); + for relay in [false, true] { + let socket = temp.path().join("ssh.sock"); + let (listener, config, _) = + ssh_server_init(&socket, &None, false, key.clone()).unwrap(); + let server = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let config = config.clone(); + tokio::spawn(async move { + let handler = SshHandler::new( + Arc::new(TestLoopbackConnector), + Arc::new(RejectingExec), + None, + ); + if let Ok(session) = + russh::server::run_stream(config, stream, handler).await + { + let _ = session.await; + } + }); + } + }); + let direct = tokio::net::UnixStream::connect(&socket).await.unwrap(); + let stream: openshell_isolation_interface::contract::BoundaryDuplexStream = if relay { + let (client, mut tunnel) = tokio::io::duplex(64 * 1024); + tokio::spawn(async move { + let mut direct = direct; + let _ = tokio::io::copy_bidirectional(&mut tunnel, &mut direct).await; + }); + Box::new(client) + } else { + Box::new(direct) + }; + let mut client = russh::client::connect_stream( + Arc::new(russh::client::Config::default()), + stream, + PinnedServerKey(fingerprint.clone()), + ) + .await + .unwrap(); + assert!(matches!( + client.authenticate_none("sandbox").await.unwrap(), + russh::client::AuthResult::Success + )); + client + .disconnect(russh::Disconnect::ByApplication, "done", "") + .await + .unwrap(); + let stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); + assert!( + russh::client::connect_stream( + Arc::new(russh::client::Config::default()), + stream, + PinnedServerKey("SHA256:wrong".to_string()) + ) + .await + .is_err() + ); + server.abort(); + let _ = server.await; + } + } + impl russh::client::Handler for AcceptAnyServerKey { type Error = russh::Error; @@ -1814,7 +1928,13 @@ mod tests { set_file_mode(&parent, 0o775); let socket = parent.join("ssh.sock"); - let (listener, _, _) = ssh_server_init(&socket, &None, false).unwrap(); + let (listener, _, _) = ssh_server_init( + &socket, + &None, + false, + PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(), + ) + .unwrap(); drop(listener); assert_eq!(file_mode(&parent), 0o700); @@ -1830,7 +1950,13 @@ mod tests { set_file_mode(&parent, 0o775); let socket = parent.join("ssh.sock"); - let (listener, _, _) = ssh_server_init(&socket, &None, true).unwrap(); + let (listener, _, _) = ssh_server_init( + &socket, + &None, + true, + PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(), + ) + .unwrap(); drop(listener); assert_eq!(file_mode(&parent), 0o775); @@ -1841,7 +1967,13 @@ mod tests { #[tokio::test] async fn ssh_server_abstract_socket_cannot_be_replaced_while_bound() { let socket = PathBuf::from(format!("@openshell-ssh-test-{}", uuid::Uuid::new_v4())); - let (listener, _, _) = ssh_server_init(&socket, &None, true).unwrap(); + let (listener, _, _) = ssh_server_init( + &socket, + &None, + true, + PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(), + ) + .unwrap(); assert!( !socket.exists(), diff --git a/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs b/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs index f326c63cf2..3d58308562 100644 --- a/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs +++ b/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs @@ -111,7 +111,13 @@ impl Connection { async fn with_window(main: Arc, window: Option) -> Self { let dir = tempfile::tempdir().unwrap(); let socket = dir.path().join("ssh.sock"); - let (listener, mut config, _) = ssh_server_init(&socket, &None, false).unwrap(); + let (listener, mut config, _) = ssh_server_init( + &socket, + &None, + false, + PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(), + ) + .unwrap(); // Preserve whether production enables probes. Only shorten the interval. let config_mut = Arc::get_mut(&mut config).unwrap(); config_mut.keepalive_interval = config_mut.keepalive_interval.map(|_| TEST_INTERVAL); @@ -407,7 +413,13 @@ async fn waiting_writer_eof_cancels_acquisition() { #[tokio::test] async fn production_config_probes_before_the_receive_deadline() { let dir = tempfile::tempdir().unwrap(); - let (_, config, _) = ssh_server_init(&dir.path().join("ssh.sock"), &None, false).unwrap(); + let (_, config, _) = ssh_server_init( + &dir.path().join("ssh.sock"), + &None, + false, + PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(), + ) + .unwrap(); assert_eq!(config.keepalive_interval, Some(Duration::from_secs(15))); assert_eq!(config.keepalive_max, 3); assert_eq!(SSH_PEER_TIMEOUT, Duration::from_mins(1)); diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 63c5302673..9b8cab8ead 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -683,6 +683,13 @@ pub async fn run_sandbox( )); } let sandbox_bearer = openshell_core::grpc_client::install_supervisor_auth_bundle(&auth_bundle)?; + let ssh_host_key = if ssh_socket_path.is_some() { + Some(openshell_supervisor_process::ssh::parse_host_key( + auth_bundle.ssh_host_private_key.as_ref(), + )?) + } else { + None + }; let (image_yaml, invalid_image) = openshell_sandbox_backend::OpenShellRuntimeBackend::discover_policy( runtime_descriptor.clone(), @@ -1168,6 +1175,7 @@ pub async fn run_sandbox( running.loopback_connector(), agent.clone(), Some(supervisor_session_updates), + ssh_host_key, ) .await?; info!(backend = %backend_name, "Control-mode access plane started"); diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 8194b0df5b..23ecbb29cf 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -799,6 +799,29 @@ openshell sandbox create --from registry.example.com/your-org/claude-agent:lates ## SSH Config +Each sandbox has its own SSH host key. The key stays the same when the +supervisor or container restarts, a pod is rescheduled, or the sandbox is +stopped and started. Deleting the sandbox removes its key. Creating another +sandbox with the same name creates a different key. + +The gateway returns the public fingerprint in `Sandbox.host_key_fingerprint` +from sandbox creation, lookup, and listing, and in +`CreateSshSessionResponse.host_key_fingerprint`. The value uses OpenSSH's +`SHA256:` format. Read it over an authenticated gateway connection, +pin it to the sandbox ID, and reject SSH connections that present a different +fingerprint. This also applies to clients that connect directly to the +supervisor's SSH socket. + +The gateway keeps the private key in its credential store and delivers it +only to the supervisor. The sandbox workload cannot read the key. Drivers, +gateway, and supervisor must use the matching OpenShell release. Older +runtimes can return an empty fingerprint; an empty value cannot verify an +SSH server's identity. + +The CLI and TUI continue to use their existing SSH connection settings. +Exposing the fingerprint does not enable automatic host-key verification in +those clients. + Generate an SSH config entry for a sandbox so tools like VS Code Remote-SSH can connect directly: ```shell diff --git a/e2e/rust/tests/ssh_host_identity.rs b/e2e/rust/tests/ssh_host_identity.rs new file mode 100644 index 0000000000..6872b898a0 --- /dev/null +++ b/e2e/rust/tests/ssh_host_identity.rs @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e")] + +use std::path::Path; +use std::time::Duration; + +use openshell_e2e::harness::cli::{ + run_cli, wait_for_healthy, wait_for_sandbox_exec_contains, wait_for_sandbox_phase, +}; +use openshell_e2e::harness::gateway::ManagedGateway; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::sandbox::SandboxGuard; + +async fn fingerprint(name: &str) -> String { + let (output, code) = run_cli(&["sandbox", "get", name, "--output", "json"]).await; + assert_eq!(code, 0, "sandbox get failed: {output}"); + let sandbox: serde_json::Value = serde_json::from_str(&strip_ansi(&output)).unwrap(); + let fingerprint = sandbox["host_key_fingerprint"] + .as_str() + .unwrap() + .to_string(); + assert!(fingerprint.starts_with("SHA256:")); + let (output, code) = run_cli(&["sandbox", "list", "--output", "json"]).await; + assert_eq!(code, 0, "sandbox list failed: {output}"); + let list: serde_json::Value = serde_json::from_str(&strip_ansi(&output)).unwrap(); + let listed = list["sandboxes"] + .as_array() + .unwrap() + .iter() + .find(|sandbox| sandbox["name"] == name) + .unwrap(); + assert_eq!(listed["host_key_fingerprint"], fingerprint); + fingerprint +} + +async fn pinned_ssh(config: &Path, known_hosts: &Path, name: &str, first: bool) { + let output = tokio::time::timeout( + Duration::from_secs(30), + tokio::process::Command::new("ssh") + .kill_on_drop(true) + .arg("-F") + .arg(config) + .arg("-o") + .arg(if first { + "StrictHostKeyChecking=accept-new" + } else { + "StrictHostKeyChecking=yes" + }) + .arg("-o") + .arg(format!("UserKnownHostsFile={}", known_hosts.display())) + .arg("-o") + .arg("BatchMode=yes") + .arg(format!("openshell-{name}.default")) + .args(["printf", "ssh-identity-ok"]) + .output(), + ) + .await + .unwrap() + .unwrap(); + assert!( + output.status.success(), + "pinned SSH failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout), "ssh-identity-ok"); +} + +#[tokio::test] +async fn ssh_host_identity_survives_restarts_and_changes_after_recreation() { + let mut sandbox = SandboxGuard::create(&[]).await.unwrap(); + let name = sandbox.name.clone(); + let original = fingerprint(&name).await; + let directory = tempfile::tempdir().unwrap(); + let config = directory.path().join("ssh_config"); + let known_hosts = directory.path().join("known_hosts"); + let (output, code) = run_cli(&["sandbox", "ssh-config", &name]).await; + assert_eq!(code, 0, "SSH config failed: {output}"); + std::fs::write(&config, output).unwrap(); + pinned_ssh(&config, &known_hosts, &name, true).await; + let key = tokio::process::Command::new("ssh-keygen") + .args(["-l", "-E", "sha256", "-f"]) + .arg(&known_hosts) + .output() + .await + .unwrap(); + assert!(key.status.success()); + assert!( + String::from_utf8_lossy(&key.stdout) + .split_whitespace() + .any(|word| word == original) + ); + sandbox + .exec(&["sh", "-c", "test ! -r /.openshell/supervisor/auth.json"]) + .await + .unwrap(); + + let (output, code) = run_cli(&["sandbox", "stop", &name]).await; + assert_eq!(code, 0, "stop failed: {output}"); + wait_for_sandbox_phase(&name, "Stopped", Duration::from_secs(60)) + .await + .unwrap(); + assert_eq!(fingerprint(&name).await, original); + let (output, code) = run_cli(&["sandbox", "start", &name]).await; + assert_eq!(code, 0, "start failed: {output}"); + wait_for_sandbox_phase(&name, "Ready", Duration::from_secs(120)) + .await + .unwrap(); + assert_eq!(fingerprint(&name).await, original); + pinned_ssh(&config, &known_hosts, &name, false).await; + + if let Some(gateway) = ManagedGateway::from_env().unwrap() { + gateway.stop().unwrap(); + gateway.start().unwrap(); + wait_for_healthy(Duration::from_secs(120)).await.unwrap(); + wait_for_sandbox_phase(&name, "Ready", Duration::from_secs(120)) + .await + .unwrap(); + wait_for_sandbox_exec_contains( + &name, + &["printf", "ssh-identity-ready"], + "ssh-identity-ready", + Duration::from_secs(240), + ) + .await + .unwrap(); + assert_eq!(fingerprint(&name).await, original); + pinned_ssh(&config, &known_hosts, &name, false).await; + } else { + eprintln!("Skipping gateway restart check: this run uses an existing gateway"); + } + + sandbox.cleanup().await; + let mut replacement = SandboxGuard::create(&["--name", &name]).await.unwrap(); + assert_ne!(fingerprint(&name).await, original); + replacement.cleanup().await; +} diff --git a/proto/openshell.proto b/proto/openshell.proto index 83ffbe75ad..a62e7ae48b 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -1006,6 +1006,10 @@ message Sandbox { SandboxSpec spec = 2; // Latest user-facing observed status derived by the gateway. SandboxStatus status = 3; + // Read-only SSH host identity in OpenSSH SHA256: form. Stable for + // this sandbox's lifetime, including runtime restarts. Empty on legacy + // runtimes that do not provide a gateway-managed SSH identity. + string host_key_fingerprint = 6; // Read-only provenance for sandboxes created from a reusable workload template. SandboxWorkloadTemplateProvenance created_from_workload_template = 20; @@ -1753,7 +1757,8 @@ message CreateSshSessionResponse { // Gateway scheme. Must be exactly "http" or "https". string gateway_scheme = 5; - // Optional host key fingerprint. If non-empty, [A-Za-z0-9:+/=-] only. + // Expected sandbox SSH host identity in OpenSSH SHA256: form. + // Stable for the sandbox's lifetime. Empty only for legacy runtimes. string host_key_fingerprint = 7; // Absolute expiry. Absence means no expiry. diff --git a/python/openshell/sandbox.py b/python/openshell/sandbox.py index 485af39d04..745c31e81a 100644 --- a/python/openshell/sandbox.py +++ b/python/openshell/sandbox.py @@ -521,6 +521,8 @@ class SandboxRef: service_urls: Mapping[str, str] = field( default_factory=_ImmutableLabels, compare=False ) + # Stable SSH host identity; absent on older runtimes. + host_key_fingerprint: str | None = None def __post_init__(self) -> None: object.__setattr__(self, "labels", _ImmutableLabels(self.labels)) @@ -1796,6 +1798,7 @@ def _sandbox_ref( labels=sandbox.metadata.labels if sandbox.metadata else {}, created_from_workload_template=provenance, service_urls=service_urls or {}, + host_key_fingerprint=sandbox.host_key_fingerprint or None, ) diff --git a/python/openshell/sandbox_test.py b/python/openshell/sandbox_test.py index 3b744451cb..40fc7c205f 100644 --- a/python/openshell/sandbox_test.py +++ b/python/openshell/sandbox_test.py @@ -2728,6 +2728,13 @@ def test_sandbox_ref_retains_gateway_labels() -> None: assert dict(ref.labels) == {"aiq": "deep-research", "env": "dev"} +def test_sandbox_ref_exposes_host_identity_and_accepts_legacy_response() -> None: + proto = _make_sandbox_proto("sandbox-1", "job-1") + assert _sandbox_ref(proto).host_key_fingerprint is None + proto.host_key_fingerprint = "SHA256:expected" + assert _sandbox_ref(proto).host_key_fingerprint == "SHA256:expected" + + def test_sandbox_ref_includes_main_process_result() -> None: proto = _make_sandbox_proto("sandbox-1", "job-1") proto.status.exit_code = 0 diff --git a/sdk/go/openshell/v1/internal/converter/sandbox.go b/sdk/go/openshell/v1/internal/converter/sandbox.go index 7d81a7c828..cacd18e964 100644 --- a/sdk/go/openshell/v1/internal/converter/sandbox.go +++ b/sdk/go/openshell/v1/internal/converter/sandbox.go @@ -21,7 +21,7 @@ func SandboxFromProto(s *pb.Sandbox) *types.Sandbox { return nil } - result := &types.Sandbox{} + result := &types.Sandbox{HostKeyFingerprint: s.GetHostKeyFingerprint()} if m := s.GetMetadata(); m != nil { result.ID = m.GetId() diff --git a/sdk/go/openshell/v1/internal/converter/sandbox_test.go b/sdk/go/openshell/v1/internal/converter/sandbox_test.go index 2099b5b65b..ada0c6ee2e 100644 --- a/sdk/go/openshell/v1/internal/converter/sandbox_test.go +++ b/sdk/go/openshell/v1/internal/converter/sandbox_test.go @@ -51,6 +51,7 @@ func TestSandboxFromProto(t *testing.T) { gpuCount := uint32(2) exitCode := int32(0) proto := &pb.Sandbox{ + HostKeyFingerprint: "SHA256:expected", Metadata: &dm.ObjectMeta{ Id: "sb-1", Name: "my-sandbox", @@ -119,6 +120,7 @@ func TestSandboxFromProto(t *testing.T) { } s := SandboxFromProto(proto) + assert.Equal(t, "SHA256:expected", s.HostKeyFingerprint) require.NotNil(t, s) assert.Equal(t, "sb-1", s.ID) diff --git a/sdk/go/openshell/v1/types/sandbox.go b/sdk/go/openshell/v1/types/sandbox.go index c8e7d43ef3..8e57c14f8e 100644 --- a/sdk/go/openshell/v1/types/sandbox.go +++ b/sdk/go/openshell/v1/types/sandbox.go @@ -7,6 +7,8 @@ import "time" // Sandbox represents a sandbox instance. type Sandbox struct { + // HostKeyFingerprint is the stable SSH identity; empty on older runtimes. + HostKeyFingerprint string ID string Name string CreatedAt time.Time diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 109d2ef454..af6c2b5d27 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -2270,6 +2270,10 @@ type Sandbox struct { Spec *SandboxSpec `protobuf:"bytes,2,opt,name=spec,proto3" json:"spec,omitempty"` // Latest user-facing observed status derived by the gateway. Status *SandboxStatus `protobuf:"bytes,3,opt,name=status,proto3" json:"status,omitempty"` + // Read-only SSH host identity in OpenSSH SHA256: form. Stable for + // this sandbox's lifetime, including runtime restarts. Empty on legacy + // runtimes that do not provide a gateway-managed SSH identity. + HostKeyFingerprint string `protobuf:"bytes,6,opt,name=host_key_fingerprint,json=hostKeyFingerprint,proto3" json:"host_key_fingerprint,omitempty"` // Read-only provenance for sandboxes created from a reusable workload template. CreatedFromWorkloadTemplate *SandboxWorkloadTemplateProvenance `protobuf:"bytes,20,opt,name=created_from_workload_template,json=createdFromWorkloadTemplate,proto3" json:"created_from_workload_template,omitempty"` unknownFields protoimpl.UnknownFields @@ -2327,6 +2331,13 @@ func (x *Sandbox) GetStatus() *SandboxStatus { return nil } +func (x *Sandbox) GetHostKeyFingerprint() string { + if x != nil { + return x.HostKeyFingerprint + } + return "" +} + func (x *Sandbox) GetCreatedFromWorkloadTemplate() *SandboxWorkloadTemplateProvenance { if x != nil { return x.CreatedFromWorkloadTemplate @@ -6060,7 +6071,8 @@ type CreateSshSessionResponse struct { GatewayPort uint32 `protobuf:"varint,4,opt,name=gateway_port,json=gatewayPort,proto3" json:"gateway_port,omitempty"` // Gateway scheme. Must be exactly "http" or "https". GatewayScheme string `protobuf:"bytes,5,opt,name=gateway_scheme,json=gatewayScheme,proto3" json:"gateway_scheme,omitempty"` - // Optional host key fingerprint. If non-empty, [A-Za-z0-9:+/=-] only. + // Expected sandbox SSH host identity in OpenSSH SHA256: form. + // Stable for the sandbox's lifetime. Empty only for legacy runtimes. HostKeyFingerprint string `protobuf:"bytes,7,opt,name=host_key_fingerprint,json=hostKeyFingerprint,proto3" json:"host_key_fingerprint,omitempty"` // Absolute expiry. Absence means no expiry. ExpirationTime *timestamppb.Timestamp `protobuf:"bytes,108,opt,name=expiration_time,json=expirationTime,proto3" json:"expiration_time,omitempty"` @@ -17890,11 +17902,12 @@ const file_openshell_proto_rawDesc = "" + "\x0flimit_supported\x18\x01 \x01(\bR\x0elimitSupported\"\x95\x01\n" + "\x17GpuResourceCapabilities\x12>\n" + "\x1bdefault_selection_supported\x18\x01 \x01(\bR\x19defaultSelectionSupported\x12:\n" + - "\x19count_selection_supported\x18\x02 \x01(\bR\x17countSelectionSupported\"\xce\x02\n" + + "\x19count_selection_supported\x18\x02 \x01(\bR\x17countSelectionSupported\"\x80\x03\n" + "\aSandbox\x12>\n" + "\bmetadata\x18\x01 \x01(\v2\".openshell.datamodel.v1.ObjectMetaR\bmetadata\x12-\n" + "\x04spec\x18\x02 \x01(\v2\x19.openshell.v1.SandboxSpecR\x04spec\x123\n" + - "\x06status\x18\x03 \x01(\v2\x1b.openshell.v1.SandboxStatusR\x06status\x12t\n" + + "\x06status\x18\x03 \x01(\v2\x1b.openshell.v1.SandboxStatusR\x06status\x120\n" + + "\x14host_key_fingerprint\x18\x06 \x01(\tR\x12hostKeyFingerprint\x12t\n" + "\x1ecreated_from_workload_template\x18\x14 \x01(\v2/.openshell.v1.SandboxWorkloadTemplateProvenanceR\x1bcreatedFromWorkloadTemplateJ\x04\b\x04\x10\x05J\x04\b\x05\x10\x06R\x05phaseR\x16current_policy_version\"\x8a\x05\n" + "\vSandboxSpec\x12\x1b\n" + "\tlog_level\x18\x01 \x01(\tR\blogLevel\x12L\n" + diff --git a/sdk/typescript/src/client.test.ts b/sdk/typescript/src/client.test.ts index 178a73cc6e..45464409e4 100644 --- a/sdk/typescript/src/client.test.ts +++ b/sdk/typescript/src/client.test.ts @@ -65,6 +65,24 @@ function readySandbox( const enc = (s: string) => new TextEncoder().encode(s); +describe('sandbox SSH host identity', () => { + it('exposes the fingerprint from get and list, and accepts legacy responses', async () => { + let fingerprint = 'SHA256:expected'; + const sandbox = client({ + getSandbox: () => ({ + sandbox: { metadata: { id: 'id', name: 'work', workspace: 'default' }, hostKeyFingerprint: fingerprint }, + }), + listSandboxes: () => ({ + sandboxes: [{ metadata: { id: 'id', name: 'work', workspace: 'default' }, hostKeyFingerprint: fingerprint }], + }), + }); + expect((await sandbox.get('work')).hostKeyFingerprint).toBe(fingerprint); + expect((await sandbox.list().all())[0]?.hostKeyFingerprint).toBe(fingerprint); + fingerprint = ''; + expect((await sandbox.get('work')).hostKeyFingerprint).toBeUndefined(); + }); +}); + describe('deletion outcomes', () => { it('defaults to strict deletion and preserves accepted identity and unknown values', async () => { const flags: boolean[] = []; diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.ts index e73bccd314..f735ff9495 100644 --- a/sdk/typescript/src/client.ts +++ b/sdk/typescript/src/client.ts @@ -216,6 +216,8 @@ export interface SandboxRef { labels: Record; /** u64 rendered as a string — JS numbers can't hold it safely. */ resourceVersion: string; + /** Stable SSH host identity; absent on older runtimes. */ + hostKeyFingerprint?: string; mainProcessInstanceId?: string; exitCode?: number; createdFromWorkloadTemplate?: SandboxWorkloadTemplateProvenance; @@ -535,6 +537,7 @@ function sandboxRef(sandbox: Sandbox | undefined, serviceUrls: Record`. For configured MCP-over-HTTP endpoints, JSON output exposes each address together with `last_result` and `last_reported_at` in `endpoint_statuses`. Select the endpoint by host, path, and ports, then check the reported failure boundary. `last_reported_at` records gateway acceptance time and can advance when retained evidence is accepted after a reset. Results do not expire or prove current availability; `HttpResponseReceived` can still contain a tool error. If several paths share a host and port, a failure before the path is known remains in logs. Verify the actual operation when current tool availability matters. - On Docker Desktop, repeated `Policy fetch failed after 5 attempts` messages can mean host networking is disabled. Enable host networking in Docker diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index 3977018a01..08706cc985 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -490,6 +490,13 @@ stopped or completed. Starting a retained `Completed` or invalidates SSH sessions from the previous runtime generation. Delete remains the operation that removes retained state. +The sandbox's SSH host identity survives stop/start and runtime restarts. +API clients can read its public fingerprint from the sandbox resource or +SSH-session response and pin it to the sandbox ID. Recreating a deleted +sandbox, even under the same name, creates a different identity. See the +[sandbox SSH documentation](https://docs.nvidia.com/openshell/latest/how-it-works/sandboxes/overview.md) +for verification behavior and release compatibility. + --- ## Workflow 4: Policy Iteration Loop From a645d9fd5d61993ed8c8f3d848955e38f0e81be5 Mon Sep 17 00:00:00 2001 From: Mike Nguyen Date: Fri, 2 Oct 2026 11:49:14 +0700 Subject: [PATCH 2/2] fix(compute): clean up failed sandbox SSH identity creation Signed-off-by: Mike Nguyen --- crates/openshell-server/src/compute/mod.rs | 540 ++++++++++++++++++--- 1 file changed, 484 insertions(+), 56 deletions(-) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index a08bb31cb2..d0804a6346 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -1166,23 +1166,40 @@ impl ComputeRuntime { // The parent now owns any staged key, including if this request is // cancelled. Keep the creation guard until its public identity and // protected launch bundle have been committed. - let launch_authentication = if let Some(encoded) = launch_authentication { - let authentication = serde_json::from_slice(&encoded) - .map_err(|_| Status::internal("invalid sandbox launch authentication"))?; - let encoded = self - .encode_launch_authentication(&sandbox, authentication) - .await?; - sandbox = self - .store - .get_message::(&sandbox_id) - .await - .map_err(|_| Status::unavailable("reload sandbox SSH identity failed"))? - .ok_or_else(|| Status::aborted("sandbox was removed during creation"))?; - Some(encoded) - } else { - None - }; + let prepared = async { + if let Some(encoded) = launch_authentication { + let authentication = serde_json::from_slice(&encoded) + .map_err(|_| Status::internal("invalid sandbox launch authentication"))?; + let encoded = self + .encode_launch_authentication(&sandbox, authentication) + .await?; + sandbox = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|_| Status::unavailable("reload sandbox SSH identity failed"))? + .ok_or_else(|| Status::aborted("sandbox was removed during creation"))?; + Ok(Some(encoded)) + } else { + Ok(None) + } + } + .await; drop(global_guard); + let launch_authentication = match prepared { + Ok(authentication) => authentication, + Err(status) => { + return Err(self + .compensate_failed_create( + &sandbox_id, + sandbox.object_name(), + lifecycle_guard, + None, + status, + ) + .await); + } + }; if let Some(token) = sandbox_token && let Some(spec) = driver_sandbox.spec.as_mut() @@ -1224,7 +1241,7 @@ impl ComputeRuntime { &sandbox_id, sandbox.object_name(), lifecycle_guard, - global_guard, + Some(global_guard), status, ) .await); @@ -1250,7 +1267,7 @@ impl ComputeRuntime { &sandbox_id, sandbox.object_name(), lifecycle_guard, - global_guard, + Some(global_guard), status, ) .await); @@ -1261,40 +1278,73 @@ impl ComputeRuntime { self.sandbox_watch_bus.notify(sandbox.object_id()); Ok(sandbox) } - Err(status) if status.code() == Code::AlreadyExists => { - self.cleanup_ssh_identity(&sandbox).await?; - let _ = self - .store - .delete(Sandbox::object_type(), sandbox.object_id()) - .await; - self.sandbox_index.remove_sandbox(sandbox.object_id()); - Err(Status::already_exists("sandbox already exists")) - } - Err(status) if status.code() == Code::FailedPrecondition => { - self.cleanup_ssh_identity(&sandbox).await?; - let _ = self - .store - .delete(Sandbox::object_type(), sandbox.object_id()) - .await; - self.sandbox_index.remove_sandbox(sandbox.object_id()); - Err(Status::failed_precondition(status.message().to_string())) - } Err(err) => { - self.cleanup_ssh_identity(&sandbox).await?; - let _ = self - .store - .delete(Sandbox::object_type(), sandbox.object_id()) - .await; - self.sandbox_index.remove_sandbox(sandbox.object_id()); - Err(Status::internal(format!( - "create sandbox failed: {}", - err.message() - ))) + let status = match err.code() { + Code::AlreadyExists => Status::already_exists("sandbox already exists"), + Code::FailedPrecondition => { + Status::failed_precondition(err.message().to_string()) + } + _ => Status::internal(format!("create sandbox failed: {}", err.message())), + }; + Err(self + .compensate_failed_create( + &sandbox_id, + sandbox.object_name(), + lifecycle_guard, + None, + status, + ) + .await) } } } async fn compensate_failed_create( + &self, + sandbox_id: &str, + sandbox_name: &str, + lifecycle_guard: SandboxLifecycleGuard, + global_guard: Option>, + original: Status, + ) -> Status { + // Once create has committed its parent, cancellation must not interrupt + // compensation before it removes that parent or persists Deleting. + let runtime = self.clone(); + let sandbox_id = sandbox_id.to_string(); + let sandbox_name = sandbox_name.to_string(); + let fallback = original.clone(); + let request_span = tracing::Span::current(); + tokio::spawn( + async move { + let global_guard = match global_guard { + Some(guard) => guard, + None => runtime.lock_global_for_lifecycle(&lifecycle_guard).await, + }; + runtime + .compensate_failed_create_inner( + &sandbox_id, + &sandbox_name, + lifecycle_guard, + global_guard, + original, + ) + .await + } + .instrument(request_span), + ) + .await + .unwrap_or_else(|_| { + Status::new( + fallback.code(), + format!( + "{}; failed-create cleanup worker failed", + fallback.message() + ), + ) + }) + } + + async fn compensate_failed_create_inner( &self, sandbox_id: &str, sandbox_name: &str, @@ -1311,7 +1361,7 @@ impl ComputeRuntime { return Status::new( original.code(), format!( - "{}; cleanup after successful create was already claimed", + "{}; cleanup after failed create was already claimed", original.message() ), ); @@ -1331,7 +1381,7 @@ impl ComputeRuntime { return Status::new( original.code(), format!( - "{}; cleanup after successful create could not claim the sandbox record: {}{}", + "{}; cleanup after failed create could not claim the sandbox record: {}{}", original.message(), error.message(), cleanup_detail @@ -1350,10 +1400,22 @@ impl ComputeRuntime { Ok(deleted) => { if deleted { // The driver accepted an asynchronous deletion. Keep the - // durable Deleting record until the watch path confirms - // that the backend is absent, matching ordinary delete - // semantics. - original + // durable Deleting record until a lookup or watch event + // confirms that the backend is absent. + match self + .cleanup_local_state_if_sandbox_absent(&lifecycle_guard, sandbox_id) + .await + { + Ok(_) => original, + Err(error) => Status::new( + original.code(), + format!( + "{}; cleanup after failed create remains pending: {}", + original.message(), + error.message() + ), + ), + } } else if self .remove_deleting_sandbox_record(&lifecycle_guard, sandbox_id) .await @@ -1363,19 +1425,20 @@ impl ComputeRuntime { Status::new( original.code(), format!( - "{}; cleanup after successful create lost ownership of the sandbox record", + "{}; cleanup after failed create remains pending", original.message() ), ) } } Err(error) => { - self.recover_failed_delete(&lifecycle_guard, &transition) - .await; + // This sandbox never finished creation. Unlike a user-requested + // delete, it must not recover back to its previous live phase. + // Keep Deleting and credential ownership for reconciliation. Status::new( original.code(), format!( - "{}; cleanup after successful create failed: {}", + "{}; cleanup after failed create remains pending: {}", original.message(), error.message() ), @@ -7600,6 +7663,8 @@ mod tests { create_started: Notify, create_release: Semaphore, create_blocked: AtomicBool, + create_calls: AtomicUsize, + create_error: TestMutex>, stop_started: Notify, stop_finished: Notify, stop_release: Semaphore, @@ -7639,6 +7704,8 @@ mod tests { create_started: Notify::new(), create_release: Semaphore::new(0), create_blocked: AtomicBool::new(false), + create_calls: AtomicUsize::new(0), + create_error: TestMutex::new(None), stop_started: Notify::new(), stop_finished: Notify::new(), stop_release: Semaphore::new(0), @@ -7884,6 +7951,7 @@ mod tests { &self, _request: Request, ) -> Result, Status> { + self.create_calls.fetch_add(1, Ordering::SeqCst); self.create_started.notify_one(); if self.create_blocked.load(Ordering::SeqCst) { self.create_release @@ -7892,6 +7960,13 @@ mod tests { .expect("create release semaphore closed") .forget(); } + let create_error = *self + .create_error + .lock() + .expect("create error lock poisoned"); + if let Some(code) = create_error { + return Err(Status::new(code, "injected create failure")); + } Ok(tonic::Response::new(CreateSandboxResponse { runtime_identity: self .runtime_identity @@ -8140,6 +8215,359 @@ mod tests { (directory, runtime) } + fn configure_test_ssh_credentials( + runtime: &mut ComputeRuntime, + ) -> crate::credentials::CredentialRuntime { + enable_runtime_identity_binding(runtime); + let config = crate::Config::new(None) + .with_credential_drivers(["test-static"]) + .with_default_credential_driver(Some("test-static")); + let credentials = crate::credentials::CredentialRuntime::from_config_with_store( + &config, + runtime.store.clone(), + ) + .unwrap(); + runtime.configure_ssh_identities(credentials.clone()); + credentials + } + + fn test_launch_authentication(sandbox: &Sandbox) -> Vec { + let identity = crate::auth::sandbox_session::PersistedSandboxIdentity::read( + &sandbox.metadata.as_ref().unwrap().annotations, + ) + .unwrap(); + serde_json::to_vec( + &test_session_authority() + .mint_persisted_launch(sandbox.object_id(), &identity) + .unwrap(), + ) + .unwrap() + } + + #[tokio::test] + async fn ssh_credential_store_failure_compensates_create_and_releases_name() { + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); + driver.set_runtime_identity("test-runtime"); + let mut runtime = test_runtime(driver.clone()).await; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-key-store-failure", + "reusable-name", + SandboxPhase::Provisioning, + ); + credentials.fail_next_store(); + + let error = runtime + .create_sandbox_authenticated( + sandbox.clone(), + None, + Some(test_launch_authentication(&sandbox)), + false, + ) + .await + .expect_err("credential failure must fail create"); + + assert_eq!(error.code(), Code::Unavailable); + assert!( + runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + ); + assert_eq!(credentials.stored_credential_count(), Some(0)); + assert_eq!(driver.create_calls.load(Ordering::SeqCst), 0); + let retry = sandbox_record( + "sb-key-store-retry", + "reusable-name", + SandboxPhase::Provisioning, + ); + runtime + .create_sandbox_authenticated( + retry.clone(), + None, + Some(test_launch_authentication(&retry)), + false, + ) + .await + .unwrap(); + assert_eq!(credentials.stored_credential_count(), Some(1)); + assert_eq!(driver.create_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn ssh_credential_delete_failure_keeps_failed_create_deleting_until_reconciliation() { + for code in [ + Code::AlreadyExists, + Code::FailedPrecondition, + Code::Unavailable, + ] { + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); + *driver.create_error.lock().unwrap() = Some(code); + let mut runtime = test_runtime(driver.clone()).await; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-key-delete-failure", + "retry-cleanup", + SandboxPhase::Provisioning, + ); + credentials.fail_next_delete(); + + let error = runtime + .create_sandbox_authenticated( + sandbox.clone(), + None, + Some(test_launch_authentication(&sandbox)), + false, + ) + .await + .expect_err("driver rejection must fail create"); + + assert_eq!( + error.code(), + if code == Code::Unavailable { + Code::Internal + } else { + code + } + ); + let retained = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(retained.phase(), SandboxPhase::Deleting as i32); + assert_eq!(credentials.stored_credential_count(), Some(1)); + runtime + .reconcile_store_with_backend(Duration::ZERO) + .await + .unwrap(); + assert!( + runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + ); + assert_eq!(credentials.stored_credential_count(), Some(0)); + } + } + + #[tokio::test] + async fn ssh_fingerprint_persistence_failure_compensates_staged_identity() { + let directory = tempfile::tempdir().unwrap(); + let database_url = format!("sqlite://{}", directory.path().join("gateway.db").display()); + let store = Arc::new(Store::connect(&database_url).await.unwrap()); + let pool = sqlx::SqlitePool::connect(&database_url).await.unwrap(); + sqlx::query( + "CREATE TRIGGER reject_test_ssh_fingerprint \ + BEFORE UPDATE OF payload ON objects \ + WHEN instr(NEW.payload, CAST('SHA256:' AS BLOB)) > 0 \ + BEGIN SELECT RAISE(ABORT, 'injected fingerprint persistence failure'); END", + ) + .execute(&pool) + .await + .unwrap(); + pool.close().await; + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); + let mut runtime = test_runtime(driver.clone()).await; + runtime.store = store; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-fingerprint-failure", + "fingerprint-failure", + SandboxPhase::Provisioning, + ); + + runtime + .create_sandbox_authenticated( + sandbox.clone(), + None, + Some(test_launch_authentication(&sandbox)), + false, + ) + .await + .expect_err("fingerprint persistence failure must fail create"); + + assert!( + runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + ); + assert_eq!(credentials.stored_credential_count(), Some(0)); + assert_eq!(driver.create_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn ssh_failed_create_backend_cleanup_error_does_not_restore_provisioning() { + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Error("injected delete failure")); + driver.set_get_outcome(ControlledGetOutcome::Error("injected lookup failure")); + *driver.create_error.lock().unwrap() = Some(Code::FailedPrecondition); + let mut runtime = test_runtime(driver.clone()).await; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-backend-cleanup-failure", + "backend-cleanup-failure", + SandboxPhase::Provisioning, + ); + + runtime + .create_sandbox_authenticated( + sandbox.clone(), + None, + Some(test_launch_authentication(&sandbox)), + false, + ) + .await + .expect_err("driver rejection must fail create"); + + let retained = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(retained.phase(), SandboxPhase::Deleting as i32); + assert_eq!(credentials.stored_credential_count(), Some(1)); + driver.set_get_outcome(ControlledGetOutcome::Missing); + runtime + .reconcile_store_with_backend(Duration::ZERO) + .await + .unwrap(); + assert!( + runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + ); + assert_eq!(credentials.stored_credential_count(), Some(0)); + } + + #[tokio::test] + async fn ssh_failed_create_compensation_survives_request_cancellation() { + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); + driver.block_delete(); + *driver.create_error.lock().unwrap() = Some(Code::FailedPrecondition); + let mut runtime = test_runtime(driver.clone()).await; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-cancel-cleanup", + "cancel-cleanup", + SandboxPhase::Provisioning, + ); + let worker_runtime = runtime.clone(); + let worker_sandbox = sandbox.clone(); + let request = tokio::spawn(async move { + worker_runtime + .create_sandbox_authenticated( + worker_sandbox.clone(), + None, + Some(test_launch_authentication(&worker_sandbox)), + false, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), driver.delete_started.notified()) + .await + .unwrap(); + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + driver.release_delete(); + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("owned cleanup must finish after caller cancellation"); + assert_eq!(credentials.stored_credential_count(), Some(0)); + } + + #[tokio::test] + async fn ssh_failed_create_compensation_survives_cancellation_while_waiting_for_lock() { + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); + driver.block_create(); + *driver.create_error.lock().unwrap() = Some(Code::FailedPrecondition); + let mut runtime = test_runtime(driver.clone()).await; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-cancel-cleanup-lock", + "cancel-cleanup-lock", + SandboxPhase::Provisioning, + ); + let worker_runtime = runtime.clone(); + let worker_sandbox = sandbox.clone(); + let request = tokio::spawn(async move { + worker_runtime + .create_sandbox_authenticated( + worker_sandbox.clone(), + None, + Some(test_launch_authentication(&worker_sandbox)), + false, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), driver.create_started.notified()) + .await + .unwrap(); + let guard = runtime.sync_lock.clone().lock_owned().await; + let held_references = Arc::strong_count(&runtime.sync_lock); + driver.release_create(); + // An additional owned reference shows cleanup has started waiting. + // The cleanup worker must be owned before waiting for this guard. + tokio::time::timeout(Duration::from_secs(5), async { + while Arc::strong_count(&runtime.sync_lock) <= held_references { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + drop(guard); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("owned cleanup must finish after cancellation while waiting for lock"); + assert_eq!(credentials.stored_credential_count(), Some(0)); + } + #[tokio::test] async fn empty_create_runtime_identity_deletes_backend_and_record() { let driver = ControlledDriver::new();