diff --git a/Cargo.lock b/Cargo.lock index c6173fee09..2e24a3c7b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4685,6 +4685,7 @@ dependencies = [ "prost-reflect", "prost-types", "protoc-bin-vendored", + "rand 0.10.2", "rand 0.9.4", "rcgen", "reqwest 0.12.28", diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 02c806c421..79b8ceb9c5 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -2793,6 +2793,7 @@ fn sandbox_to_json(sandbox: &Sandbox) -> serde_json::Value { "id": sandbox.object_id(), "name": sandbox.object_name(), "workspace": sandbox.object_workspace(), + "host_key_fingerprint": sandbox.host_key_fingerprint, "labels": labels, "annotations": annotations, "resource_version": meta.map_or(0, |m| m.resource_version), diff --git a/crates/openshell-core/src/jwt.rs b/crates/openshell-core/src/jwt.rs index d1b8d22226..0dac20fce7 100644 --- a/crates/openshell-core/src/jwt.rs +++ b/crates/openshell-core/src/jwt.rs @@ -229,6 +229,46 @@ mod session { } } + /// Private SSH key material is redacted from diagnostics and cleared on drop. + #[derive(Clone)] + pub struct SecretSshHostKey(Zeroizing); + + impl SecretSshHostKey { + #[must_use] + pub fn new(value: String) -> Self { + Self(Zeroizing::new(value)) + } + + #[must_use] + pub fn expose_secret(&self) -> &str { + &self.0 + } + } + + impl fmt::Debug for SecretSshHostKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("SecretSshHostKey([REDACTED])") + } + } + + impl Serialize for SecretSshHostKey { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.expose_secret()) + } + } + + impl<'de> Deserialize<'de> for SecretSshHostKey { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + String::deserialize(deserializer).map(Self::new) + } + } + /// Trusted launch input delivered only to `openshell-supervisor`. #[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -244,6 +284,9 @@ mod session { pub gateway_expires_at: i64, pub sandbox_token: SecretJwt, pub sandbox_expires_at: i64, + /// Never delivered to the workload. Missing only in older bundles. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ssh_host_private_key: Option, } /// Gateway-created authentication input trusted by a compute driver. @@ -329,6 +372,7 @@ mod session { .field("gateway_expires_at", &self.gateway_expires_at) .field("sandbox_token", &"[REDACTED]") .field("sandbox_expires_at", &self.sandbox_expires_at) + .field("ssh_host_private_key", &self.ssh_host_private_key) .finish() } } @@ -1034,6 +1078,7 @@ mod tests { gateway_expires_at: pair.gateway.expires_at, sandbox_token: pair.sandbox.token, sandbox_expires_at: pair.sandbox.expires_at, + ssh_host_private_key: None, }; bundle.validate().expect("non-expiring auth bundle"); } @@ -1060,6 +1105,9 @@ mod tests { gateway_expires_at: pair.gateway.expires_at, sandbox_token: pair.sandbox.token, sandbox_expires_at: pair.sandbox.expires_at, + ssh_host_private_key: Some(SecretSshHostKey::new( + "private-ssh-host-key".to_string(), + )), }; let encoded = serde_json::to_vec(&bundle).expect("serialize auth bundle"); @@ -1078,7 +1126,19 @@ mod tests { let debug = format!("{bundle:?}"); assert!(!debug.contains(bundle.gateway_token.expose_secret())); assert!(!debug.contains(bundle.sandbox_token.expose_secret())); - assert_eq!(debug.matches("[REDACTED]").count(), 2); + assert!(!debug.contains("private-ssh-host-key")); + assert_eq!(debug.matches("[REDACTED]").count(), 3); + assert_eq!( + decoded.ssh_host_private_key.unwrap().expose_secret(), + "private-ssh-host-key" + ); + let mut legacy = serde_json::to_value(&bundle).unwrap(); + legacy + .as_object_mut() + .unwrap() + .remove("ssh_host_private_key"); + let legacy: SupervisorAuthBundle = serde_json::from_value(legacy).unwrap(); + assert!(legacy.ssh_host_private_key.is_none()); } #[derive(Serialize)] diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 8aa05302e0..b2189e43fe 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -38,6 +38,7 @@ fn test_launch_authentication() -> Vec { gateway_expires_at: i64::MAX, sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), sandbox_expires_at: i64::MAX, + ssh_host_private_key: None, }, gateway_id: "gateway-test".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index d3023cddd8..146f263703 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -2062,6 +2062,7 @@ mod tests { gateway_expires_at: i64::MAX, sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), sandbox_expires_at: i64::MAX, + ssh_host_private_key: None, }, gateway_id: "gateway-test".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index ae33796ea6..3eab523746 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -445,6 +445,7 @@ mod tests { gateway_expires_at: i64::MAX, sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), sandbox_expires_at: i64::MAX, + ssh_host_private_key: None, }, gateway_id: "gateway-test".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 3fceaf0e80..1e13e97e35 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -8856,6 +8856,7 @@ mod tests { gateway_expires_at: 1, sandbox_token: SecretJwt::parse(format!("sandbox-{label}")).expect("sandbox token"), sandbox_expires_at: 1, + ssh_host_private_key: None, }, gateway_id: "gateway-a".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-sdk/src/types.rs b/crates/openshell-sdk/src/types.rs index aa2bc71bf9..e4b66942dd 100644 --- a/crates/openshell-sdk/src/types.rs +++ b/crates/openshell-sdk/src/types.rs @@ -384,6 +384,8 @@ pub struct SandboxRef { pub labels: HashMap, pub resource_version: u64, pub exit_code: Option, + /// Public OpenSSH SHA256 host identity; absent on older gateways. + pub host_key_fingerprint: Option, pub created_from_workload_template: Option, /// Service URLs returned by sandbox creation, keyed by service name. The /// empty key identifies the unnamed service. Non-create reads leave this empty. @@ -437,6 +439,8 @@ impl SandboxRef { labels: meta.labels, resource_version: meta.resource_version, exit_code, + host_key_fingerprint: (!sandbox.host_key_fingerprint.is_empty()) + .then_some(sandbox.host_key_fingerprint), created_from_workload_template, service_urls: HashMap::new(), restart_count, diff --git a/crates/openshell-sdk/tests/client_mock.rs b/crates/openshell-sdk/tests/client_mock.rs index 9506160f07..9abc5e9e54 100644 --- a/crates/openshell-sdk/tests/client_mock.rs +++ b/crates/openshell-sdk/tests/client_mock.rs @@ -131,6 +131,11 @@ fn sandbox_with_phase_ws( ..Default::default() }), created_from_workload_template, + host_key_fingerprint: if name == "pinned-identity" { + "SHA256:public-identity".to_string() + } else { + String::new() + }, } } @@ -1291,6 +1296,25 @@ async fn get_sandbox_sends_name_and_maps_phase() { assert_eq!(observed.as_deref(), Some("my-box")); } +#[tokio::test] +async fn get_sandbox_preserves_host_fingerprint_and_accepts_older_gateways() { + let endpoint = start_mock(Arc::new(MockState::default())).await; + let client = connect(&endpoint).await; + let sandbox = client.get_sandbox("pinned-identity").await.unwrap(); + assert_eq!( + sandbox.host_key_fingerprint.as_deref(), + Some("SHA256:public-identity") + ); + assert!( + client + .get_sandbox("legacy") + .await + .unwrap() + .host_key_fingerprint + .is_none() + ); +} + #[tokio::test] async fn get_sandbox_preserves_workload_template_provenance() { let state = Arc::new(MockState::default()); diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 54c44a7bed..dd7b526f23 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -106,6 +106,7 @@ glob = { workspace = true } hex = "0.4" russh = "0.62" rand = { workspace = true } +rand-ssh = { package = "rand", version = "0.10" } petname = "2" ipnet = "2" tempfile = "3" diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index a86eca1fdb..9446512c00 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -232,6 +232,7 @@ impl SandboxSessionJwtAuthority { gateway_expires_at: pair.gateway.expires_at, sandbox_token: pair.sandbox.token, sandbox_expires_at: pair.sandbox.expires_at, + ssh_host_private_key: None, }, gateway_id: self.gateway_id.clone(), verification_keys: self.verification_keys.clone(), diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 264cd77bbb..d0804a6346 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -659,6 +659,7 @@ pub struct ComputeRuntime { restart_authority: Arc>>>, restart_notify: Arc, + ssh_identities: Arc>, } pub struct SandboxSyncGuard { @@ -756,9 +757,73 @@ impl ComputeRuntime { rootfs_tar_staging, restart_authority: Arc::new(OnceLock::new()), restart_notify: Arc::new(Notify::new()), + ssh_identities: Arc::new(OnceLock::new()), }) } + pub(crate) fn configure_ssh_identities( + &self, + credentials: crate::credentials::CredentialRuntime, + ) { + let _ = self + .ssh_identities + .set(crate::ssh_identity::SshIdentityStore::new( + self.store.clone(), + credentials, + )); + } + + pub(crate) async fn prepare_ssh_identity( + &self, + sandbox: &mut Sandbox, + authentication: &mut openshell_core::jwt::SandboxLaunchAuthentication, + ) -> Result<(), Status> { + if let Some(identities) = self.ssh_identities.get() { + identities.prepare(sandbox, authentication).await?; + } + Ok(()) + } + + pub(crate) async fn cleanup_ssh_identity(&self, sandbox: &Sandbox) -> Result<(), Status> { + if let Some(identities) = self.ssh_identities.get() { + identities.delete(sandbox).await?; + } + Ok(()) + } + + async fn encode_launch_authentication( + &self, + sandbox: &Sandbox, + mut authentication: openshell_core::jwt::SandboxLaunchAuthentication, + ) -> Result, Status> { + let mut with_identity = sandbox.clone(); + self.prepare_ssh_identity(&mut with_identity, &mut authentication) + .await?; + serialize_launch_authentication(authentication) + } + + async fn encode_persisted_launch_authentication( + &self, + authority: Option<&crate::auth::sandbox_jwt::SandboxSessionJwtAuthority>, + sandbox: &Sandbox, + ) -> Result, Status> { + let Some(authority) = authority else { + return Ok(Vec::new()); + }; + let metadata = sandbox + .metadata + .as_ref() + .ok_or_else(|| Status::failed_precondition("sandbox metadata is missing"))?; + let identity = + crate::auth::sandbox_session::PersistedSandboxIdentity::read(&metadata.annotations) + .map_err(|error| Status::failed_precondition(error.to_string()))?; + self.encode_launch_authentication( + sandbox, + authority.mint_persisted_launch(sandbox.object_id(), &identity)?, + ) + .await + } + /// Serializes sandbox/provider-profile invariant checks and object writes /// across gateway replicas. /// @@ -1098,7 +1163,43 @@ impl ComputeRuntime { if let Some(metadata) = sandbox.metadata.as_mut() { metadata.resource_version = result.resource_version; } + // The parent now owns any staged key, including if this request is + // cancelled. Keep the creation guard until its public identity and + // protected launch bundle have been committed. + let prepared = async { + if let Some(encoded) = launch_authentication { + let authentication = serde_json::from_slice(&encoded) + .map_err(|_| Status::internal("invalid sandbox launch authentication"))?; + let encoded = self + .encode_launch_authentication(&sandbox, authentication) + .await?; + sandbox = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|_| Status::unavailable("reload sandbox SSH identity failed"))? + .ok_or_else(|| Status::aborted("sandbox was removed during creation"))?; + Ok(Some(encoded)) + } else { + Ok(None) + } + } + .await; drop(global_guard); + let launch_authentication = match prepared { + Ok(authentication) => authentication, + Err(status) => { + return Err(self + .compensate_failed_create( + &sandbox_id, + sandbox.object_name(), + lifecycle_guard, + None, + status, + ) + .await); + } + }; if let Some(token) = sandbox_token && let Some(spec) = driver_sandbox.spec.as_mut() @@ -1140,7 +1241,7 @@ impl ComputeRuntime { &sandbox_id, sandbox.object_name(), lifecycle_guard, - global_guard, + Some(global_guard), status, ) .await); @@ -1166,7 +1267,7 @@ impl ComputeRuntime { &sandbox_id, sandbox.object_name(), lifecycle_guard, - global_guard, + Some(global_guard), status, ) .await); @@ -1177,37 +1278,73 @@ impl ComputeRuntime { self.sandbox_watch_bus.notify(sandbox.object_id()); Ok(sandbox) } - Err(status) if status.code() == Code::AlreadyExists => { - let _ = self - .store - .delete(Sandbox::object_type(), sandbox.object_id()) - .await; - self.sandbox_index.remove_sandbox(sandbox.object_id()); - Err(Status::already_exists("sandbox already exists")) - } - Err(status) if status.code() == Code::FailedPrecondition => { - let _ = self - .store - .delete(Sandbox::object_type(), sandbox.object_id()) - .await; - self.sandbox_index.remove_sandbox(sandbox.object_id()); - Err(Status::failed_precondition(status.message().to_string())) - } Err(err) => { - let _ = self - .store - .delete(Sandbox::object_type(), sandbox.object_id()) - .await; - self.sandbox_index.remove_sandbox(sandbox.object_id()); - Err(Status::internal(format!( - "create sandbox failed: {}", - err.message() - ))) + let status = match err.code() { + Code::AlreadyExists => Status::already_exists("sandbox already exists"), + Code::FailedPrecondition => { + Status::failed_precondition(err.message().to_string()) + } + _ => Status::internal(format!("create sandbox failed: {}", err.message())), + }; + Err(self + .compensate_failed_create( + &sandbox_id, + sandbox.object_name(), + lifecycle_guard, + None, + status, + ) + .await) } } } async fn compensate_failed_create( + &self, + sandbox_id: &str, + sandbox_name: &str, + lifecycle_guard: SandboxLifecycleGuard, + global_guard: Option>, + original: Status, + ) -> Status { + // Once create has committed its parent, cancellation must not interrupt + // compensation before it removes that parent or persists Deleting. + let runtime = self.clone(); + let sandbox_id = sandbox_id.to_string(); + let sandbox_name = sandbox_name.to_string(); + let fallback = original.clone(); + let request_span = tracing::Span::current(); + tokio::spawn( + async move { + let global_guard = match global_guard { + Some(guard) => guard, + None => runtime.lock_global_for_lifecycle(&lifecycle_guard).await, + }; + runtime + .compensate_failed_create_inner( + &sandbox_id, + &sandbox_name, + lifecycle_guard, + global_guard, + original, + ) + .await + } + .instrument(request_span), + ) + .await + .unwrap_or_else(|_| { + Status::new( + fallback.code(), + format!( + "{}; failed-create cleanup worker failed", + fallback.message() + ), + ) + }) + } + + async fn compensate_failed_create_inner( &self, sandbox_id: &str, sandbox_name: &str, @@ -1224,7 +1361,7 @@ impl ComputeRuntime { return Status::new( original.code(), format!( - "{}; cleanup after successful create was already claimed", + "{}; cleanup after failed create was already claimed", original.message() ), ); @@ -1244,7 +1381,7 @@ impl ComputeRuntime { return Status::new( original.code(), format!( - "{}; cleanup after successful create could not claim the sandbox record: {}{}", + "{}; cleanup after failed create could not claim the sandbox record: {}{}", original.message(), error.message(), cleanup_detail @@ -1263,10 +1400,22 @@ impl ComputeRuntime { Ok(deleted) => { if deleted { // The driver accepted an asynchronous deletion. Keep the - // durable Deleting record until the watch path confirms - // that the backend is absent, matching ordinary delete - // semantics. - original + // durable Deleting record until a lookup or watch event + // confirms that the backend is absent. + match self + .cleanup_local_state_if_sandbox_absent(&lifecycle_guard, sandbox_id) + .await + { + Ok(_) => original, + Err(error) => Status::new( + original.code(), + format!( + "{}; cleanup after failed create remains pending: {}", + original.message(), + error.message() + ), + ), + } } else if self .remove_deleting_sandbox_record(&lifecycle_guard, sandbox_id) .await @@ -1276,19 +1425,20 @@ impl ComputeRuntime { Status::new( original.code(), format!( - "{}; cleanup after successful create lost ownership of the sandbox record", + "{}; cleanup after failed create remains pending", original.message() ), ) } } Err(error) => { - self.recover_failed_delete(&lifecycle_guard, &transition) - .await; + // This sandbox never finished creation. Unlike a user-requested + // delete, it must not recover back to its previous live phase. + // Keep Deleting and credential ownership for reconciliation. Status::new( original.code(), format!( - "{}; cleanup after successful create failed: {}", + "{}; cleanup after failed create remains pending: {}", original.message(), error.message() ), @@ -1579,9 +1729,16 @@ impl ComputeRuntime { // Acquiring the lifecycle gate proves that no local worker still // owns this transition. Retry the idempotent driver operation // with the identity committed by the original transition. - let authentication = - serialize_persisted_launch_authentication(authority, ¤t)?; - break (current.clone(), current, authentication); + let authentication = self + .encode_persisted_launch_authentication(authority, ¤t) + .await?; + let refreshed = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|_| Status::unavailable("reload sandbox SSH identity failed"))? + .ok_or_else(|| Status::not_found("sandbox was deleted"))?; + break (refreshed.clone(), refreshed, authentication); } let previous = current.clone(); @@ -1594,11 +1751,14 @@ impl ComputeRuntime { }; let launch_authentication = if let (Some(authority), Some(identity)) = (authority, next_identity.as_ref()) { - serialize_launch_authentication( + self.encode_launch_authentication( + ¤t, authority.mint_persisted_launch(current.object_id(), identity)?, - )? + ) + .await? } else { - serialize_persisted_launch_authentication(authority, ¤t)? + self.encode_persisted_launch_authentication(authority, ¤t) + .await? }; let expected_resource_version = sandbox_resource_version(¤t); let next_identity_for_update = next_identity.clone(); @@ -3774,7 +3934,9 @@ impl ComputeRuntime { return Ok(()); }; - let launch_authentication = serialize_persisted_launch_authentication(authority, &armed) + let launch_authentication = self + .encode_persisted_launch_authentication(authority, &armed) + .await .map_err(|status| status.to_string())?; let generation_id = sandbox_runtime_generation(&armed)?.into_string(); let expected_runtime_identity = sandbox_compute_runtime_identity(¤t); @@ -4811,7 +4973,20 @@ impl ComputeRuntime { // call itself, only on the (instant, non-blocking) decision to // make it. self.spawn_driver_sandbox_cleanup(sandbox.object_id(), sandbox.object_name()); - self.cleanup_sandbox_owned_records(sandbox).await?; + // Fence a delayed identity preparer before reclaiming its key. + // A watch deletion has no request-side Deleting transition. + let deleting = self + .store + .update_message_cas::( + sandbox.object_id(), + sandbox_resource_version(sandbox), + |sandbox| { + sandbox.set_phase(SandboxPhase::Deleting.into()); + }, + ) + .await + .map_err(|error| error.to_string())?; + self.cleanup_sandbox_owned_records(&deleting).await?; } let _ = self @@ -4835,6 +5010,9 @@ impl ComputeRuntime { } async fn cleanup_sandbox_owned_records(&self, sandbox: &Sandbox) -> Result<(), String> { + self.cleanup_ssh_identity(sandbox) + .await + .map_err(|error| error.to_string())?; self.cleanup_sandbox_ssh_sessions(sandbox.object_id(), sandbox.object_workspace()) .await?; self.cleanup_sandbox_service_endpoints(sandbox.object_id(), sandbox.object_workspace()) @@ -6596,25 +6774,6 @@ fn next_runtime_identity( }) } -fn serialize_persisted_launch_authentication( - authority: Option<&crate::auth::sandbox_jwt::SandboxSessionJwtAuthority>, - sandbox: &Sandbox, -) -> Result, Status> { - let Some(authority) = authority else { - return Ok(Vec::new()); - }; - let metadata = sandbox - .metadata - .as_ref() - .ok_or_else(|| Status::failed_precondition("sandbox metadata is missing"))?; - let identity = - crate::auth::sandbox_session::PersistedSandboxIdentity::read(&metadata.annotations) - .map_err(|error| Status::failed_precondition(error.to_string()))?; - serialize_launch_authentication( - authority.mint_persisted_launch(sandbox.object_id(), &identity)?, - ) -} - fn serialize_launch_authentication( authentication: openshell_core::jwt::SandboxLaunchAuthentication, ) -> Result, Status> { @@ -6935,6 +7094,7 @@ pub fn new_test_runtime_with_driver( rootfs_tar_staging: Arc::new(rootfs_tar::RootfsTarStagingRegistry::disabled()), restart_authority: Arc::new(OnceLock::new()), restart_notify: Arc::new(Notify::new()), + ssh_identities: Arc::new(OnceLock::new()), } } @@ -7503,6 +7663,8 @@ mod tests { create_started: Notify, create_release: Semaphore, create_blocked: AtomicBool, + create_calls: AtomicUsize, + create_error: TestMutex>, stop_started: Notify, stop_finished: Notify, stop_release: Semaphore, @@ -7542,6 +7704,8 @@ mod tests { create_started: Notify::new(), create_release: Semaphore::new(0), create_blocked: AtomicBool::new(false), + create_calls: AtomicUsize::new(0), + create_error: TestMutex::new(None), stop_started: Notify::new(), stop_finished: Notify::new(), stop_release: Semaphore::new(0), @@ -7787,6 +7951,7 @@ mod tests { &self, _request: Request, ) -> Result, Status> { + self.create_calls.fetch_add(1, Ordering::SeqCst); self.create_started.notify_one(); if self.create_blocked.load(Ordering::SeqCst) { self.create_release @@ -7795,6 +7960,13 @@ mod tests { .expect("create release semaphore closed") .forget(); } + let create_error = *self + .create_error + .lock() + .expect("create error lock poisoned"); + if let Some(code) = create_error { + return Err(Status::new(code, "injected create failure")); + } Ok(tonic::Response::new(CreateSandboxResponse { runtime_identity: self .runtime_identity @@ -7984,6 +8156,7 @@ mod tests { rootfs_tar_staging: Arc::new(rootfs_tar::RootfsTarStagingRegistry::disabled()), restart_authority: Arc::new(OnceLock::new()), restart_notify: Arc::new(Notify::new()), + ssh_identities: Arc::new(OnceLock::new()), } } @@ -8042,6 +8215,359 @@ mod tests { (directory, runtime) } + fn configure_test_ssh_credentials( + runtime: &mut ComputeRuntime, + ) -> crate::credentials::CredentialRuntime { + enable_runtime_identity_binding(runtime); + let config = crate::Config::new(None) + .with_credential_drivers(["test-static"]) + .with_default_credential_driver(Some("test-static")); + let credentials = crate::credentials::CredentialRuntime::from_config_with_store( + &config, + runtime.store.clone(), + ) + .unwrap(); + runtime.configure_ssh_identities(credentials.clone()); + credentials + } + + fn test_launch_authentication(sandbox: &Sandbox) -> Vec { + let identity = crate::auth::sandbox_session::PersistedSandboxIdentity::read( + &sandbox.metadata.as_ref().unwrap().annotations, + ) + .unwrap(); + serde_json::to_vec( + &test_session_authority() + .mint_persisted_launch(sandbox.object_id(), &identity) + .unwrap(), + ) + .unwrap() + } + + #[tokio::test] + async fn ssh_credential_store_failure_compensates_create_and_releases_name() { + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); + driver.set_runtime_identity("test-runtime"); + let mut runtime = test_runtime(driver.clone()).await; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-key-store-failure", + "reusable-name", + SandboxPhase::Provisioning, + ); + credentials.fail_next_store(); + + let error = runtime + .create_sandbox_authenticated( + sandbox.clone(), + None, + Some(test_launch_authentication(&sandbox)), + false, + ) + .await + .expect_err("credential failure must fail create"); + + assert_eq!(error.code(), Code::Unavailable); + assert!( + runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + ); + assert_eq!(credentials.stored_credential_count(), Some(0)); + assert_eq!(driver.create_calls.load(Ordering::SeqCst), 0); + let retry = sandbox_record( + "sb-key-store-retry", + "reusable-name", + SandboxPhase::Provisioning, + ); + runtime + .create_sandbox_authenticated( + retry.clone(), + None, + Some(test_launch_authentication(&retry)), + false, + ) + .await + .unwrap(); + assert_eq!(credentials.stored_credential_count(), Some(1)); + assert_eq!(driver.create_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn ssh_credential_delete_failure_keeps_failed_create_deleting_until_reconciliation() { + for code in [ + Code::AlreadyExists, + Code::FailedPrecondition, + Code::Unavailable, + ] { + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); + *driver.create_error.lock().unwrap() = Some(code); + let mut runtime = test_runtime(driver.clone()).await; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-key-delete-failure", + "retry-cleanup", + SandboxPhase::Provisioning, + ); + credentials.fail_next_delete(); + + let error = runtime + .create_sandbox_authenticated( + sandbox.clone(), + None, + Some(test_launch_authentication(&sandbox)), + false, + ) + .await + .expect_err("driver rejection must fail create"); + + assert_eq!( + error.code(), + if code == Code::Unavailable { + Code::Internal + } else { + code + } + ); + let retained = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(retained.phase(), SandboxPhase::Deleting as i32); + assert_eq!(credentials.stored_credential_count(), Some(1)); + runtime + .reconcile_store_with_backend(Duration::ZERO) + .await + .unwrap(); + assert!( + runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + ); + assert_eq!(credentials.stored_credential_count(), Some(0)); + } + } + + #[tokio::test] + async fn ssh_fingerprint_persistence_failure_compensates_staged_identity() { + let directory = tempfile::tempdir().unwrap(); + let database_url = format!("sqlite://{}", directory.path().join("gateway.db").display()); + let store = Arc::new(Store::connect(&database_url).await.unwrap()); + let pool = sqlx::SqlitePool::connect(&database_url).await.unwrap(); + sqlx::query( + "CREATE TRIGGER reject_test_ssh_fingerprint \ + BEFORE UPDATE OF payload ON objects \ + WHEN instr(NEW.payload, CAST('SHA256:' AS BLOB)) > 0 \ + BEGIN SELECT RAISE(ABORT, 'injected fingerprint persistence failure'); END", + ) + .execute(&pool) + .await + .unwrap(); + pool.close().await; + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); + let mut runtime = test_runtime(driver.clone()).await; + runtime.store = store; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-fingerprint-failure", + "fingerprint-failure", + SandboxPhase::Provisioning, + ); + + runtime + .create_sandbox_authenticated( + sandbox.clone(), + None, + Some(test_launch_authentication(&sandbox)), + false, + ) + .await + .expect_err("fingerprint persistence failure must fail create"); + + assert!( + runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + ); + assert_eq!(credentials.stored_credential_count(), Some(0)); + assert_eq!(driver.create_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn ssh_failed_create_backend_cleanup_error_does_not_restore_provisioning() { + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Error("injected delete failure")); + driver.set_get_outcome(ControlledGetOutcome::Error("injected lookup failure")); + *driver.create_error.lock().unwrap() = Some(Code::FailedPrecondition); + let mut runtime = test_runtime(driver.clone()).await; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-backend-cleanup-failure", + "backend-cleanup-failure", + SandboxPhase::Provisioning, + ); + + runtime + .create_sandbox_authenticated( + sandbox.clone(), + None, + Some(test_launch_authentication(&sandbox)), + false, + ) + .await + .expect_err("driver rejection must fail create"); + + let retained = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(retained.phase(), SandboxPhase::Deleting as i32); + assert_eq!(credentials.stored_credential_count(), Some(1)); + driver.set_get_outcome(ControlledGetOutcome::Missing); + runtime + .reconcile_store_with_backend(Duration::ZERO) + .await + .unwrap(); + assert!( + runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + ); + assert_eq!(credentials.stored_credential_count(), Some(0)); + } + + #[tokio::test] + async fn ssh_failed_create_compensation_survives_request_cancellation() { + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); + driver.block_delete(); + *driver.create_error.lock().unwrap() = Some(Code::FailedPrecondition); + let mut runtime = test_runtime(driver.clone()).await; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-cancel-cleanup", + "cancel-cleanup", + SandboxPhase::Provisioning, + ); + let worker_runtime = runtime.clone(); + let worker_sandbox = sandbox.clone(); + let request = tokio::spawn(async move { + worker_runtime + .create_sandbox_authenticated( + worker_sandbox.clone(), + None, + Some(test_launch_authentication(&worker_sandbox)), + false, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), driver.delete_started.notified()) + .await + .unwrap(); + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + driver.release_delete(); + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("owned cleanup must finish after caller cancellation"); + assert_eq!(credentials.stored_credential_count(), Some(0)); + } + + #[tokio::test] + async fn ssh_failed_create_compensation_survives_cancellation_while_waiting_for_lock() { + let driver = ControlledDriver::new(); + driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); + driver.block_create(); + *driver.create_error.lock().unwrap() = Some(Code::FailedPrecondition); + let mut runtime = test_runtime(driver.clone()).await; + let credentials = configure_test_ssh_credentials(&mut runtime); + let sandbox = sandbox_record( + "sb-cancel-cleanup-lock", + "cancel-cleanup-lock", + SandboxPhase::Provisioning, + ); + let worker_runtime = runtime.clone(); + let worker_sandbox = sandbox.clone(); + let request = tokio::spawn(async move { + worker_runtime + .create_sandbox_authenticated( + worker_sandbox.clone(), + None, + Some(test_launch_authentication(&worker_sandbox)), + false, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), driver.create_started.notified()) + .await + .unwrap(); + let guard = runtime.sync_lock.clone().lock_owned().await; + let held_references = Arc::strong_count(&runtime.sync_lock); + driver.release_create(); + // An additional owned reference shows cleanup has started waiting. + // The cleanup worker must be owned before waiting for this guard. + tokio::time::timeout(Duration::from_secs(5), async { + while Arc::strong_count(&runtime.sync_lock) <= held_references { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + drop(guard); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .is_none() + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("owned cleanup must finish after cancellation while waiting for lock"); + assert_eq!(credentials.stored_credential_count(), Some(0)); + } + #[tokio::test] async fn empty_create_runtime_identity_deletes_backend_and_record() { let driver = ControlledDriver::new(); diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 074b2590e0..4378b239a8 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -574,6 +574,7 @@ async fn handle_create_sandbox_inner( let now_ms = current_time_ms(); let mut sandbox = Sandbox { + host_key_fingerprint: String::new(), metadata: Some(ObjectMeta { id: id.clone(), name: name.clone(), @@ -1745,7 +1746,7 @@ async fn handle_start_sandbox_inner( })) } -pub fn mint_persisted_authentication( +pub async fn mint_persisted_authentication( state: &ServerState, sandbox: &Sandbox, ) -> Result { @@ -1760,7 +1761,13 @@ pub fn mint_persisted_authentication( let identity = crate::auth::sandbox_session::PersistedSandboxIdentity::read(&metadata.annotations) .map_err(|error| Status::failed_precondition(error.to_string()))?; - authority.mint_persisted_launch(sandbox.object_id(), &identity) + let mut authentication = authority.mint_persisted_launch(sandbox.object_id(), &identity)?; + let mut with_identity = sandbox.clone(); + state + .compute + .prepare_ssh_identity(&mut with_identity, &mut authentication) + .await?; + Ok(authentication) } async fn providers_for_sandbox( @@ -3117,7 +3124,7 @@ pub(super) async fn handle_create_ssh_session( gateway_host, gateway_port: gateway_port.into(), gateway_scheme: scheme.to_string(), - host_key_fingerprint: String::new(), + host_key_fingerprint: sandbox.host_key_fingerprint.clone(), expiration_time: openshell_core::time::optional_timestamp_from_legacy_millis(expires_at_ms) .map_err(|error| Status::internal(error.to_string()))?, })) @@ -8342,11 +8349,9 @@ mod tests { #[tokio::test] async fn concurrent_create_ssh_session_prevents_duplicate_tokens() { let state = test_server_state().await; - state - .store - .put_message(&test_sandbox("work", Vec::new())) - .await - .unwrap(); + let mut sandbox = test_sandbox("work", Vec::new()); + sandbox.host_key_fingerprint = "SHA256:public-host-identity".to_string(); + state.store.put_message(&sandbox).await.unwrap(); // Both requests try to create sessions for the same sandbox // The token generation is random, so we can't force a collision, @@ -8386,8 +8391,12 @@ mod tests { assert!(result1.is_ok(), "first create should succeed"); assert!(result2.is_ok(), "second create should succeed"); - let token1 = result1.unwrap().into_inner().token; - let token2 = result2.unwrap().into_inner().token; + let response1 = result1.unwrap().into_inner(); + let response2 = result2.unwrap().into_inner(); + assert_eq!(response1.host_key_fingerprint, sandbox.host_key_fingerprint); + assert_eq!(response2.host_key_fingerprint, sandbox.host_key_fingerprint); + let token1 = response1.token; + let token2 = response2.token; // Tokens must be different assert_ne!(token1, token2, "tokens should be unique"); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 72efa01fbf..16b0d21632 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -38,6 +38,7 @@ mod readiness; mod sandbox_index; mod sandbox_watch; mod service_routing; +mod ssh_identity; mod ssh_sessions; mod storage_proto; mod supervisor_owner; @@ -414,6 +415,7 @@ impl ServerState { credentials: credentials::CredentialRuntime, ) -> Self { let replica_id = compute::lease::replica_id(); + compute.configure_ssh_identities(credentials.clone()); let peer_endpoint = derive_peer_endpoint(&config); let grpc_rate_limiter = multiplex::GrpcRateLimiter::from_config(&config); let admin_role = config @@ -970,6 +972,7 @@ pub(crate) async fn run_server( return Ok(Vec::new()); } let authentication = grpc::mint_persisted_authentication(&state, &sandbox) + .await .map_err(|error| error.to_string())?; serde_json::to_vec(&authentication) .map_err(|error| format!("encode launch authentication: {error}")) diff --git a/crates/openshell-server/src/persistence/mod.rs b/crates/openshell-server/src/persistence/mod.rs index 2513e80c10..1721cf7931 100644 --- a/crates/openshell-server/src/persistence/mod.rs +++ b/crates/openshell-server/src/persistence/mod.rs @@ -299,6 +299,19 @@ impl Store { } } + /// Independent of the cross-object lock: creation already holds that + /// lock when it provisions a supervisor's durable SSH identity. + pub(crate) async fn acquire_ssh_identity_mutation_guard( + &self, + ) -> PersistenceResult { + match self { + Self::Postgres(store) => Ok(DistributedMutationGuard { + _postgres: Some(store.acquire_mutation_lock(0x4f53_5348_484f_5354).await?), + }), + Self::Sqlite(_) => Ok(DistributedMutationGuard { _postgres: None }), + } + } + /// Connect to a persistence store based on the database URL. pub async fn connect(url: &str) -> CoreResult { if url.starts_with("postgres://") || url.starts_with("postgresql://") { diff --git a/crates/openshell-server/src/persistence/postgres.rs b/crates/openshell-server/src/persistence/postgres.rs index 14bae6dfb1..22a86f9f3e 100644 --- a/crates/openshell-server/src/persistence/postgres.rs +++ b/crates/openshell-server/src/persistence/postgres.rs @@ -116,6 +116,14 @@ impl PostgresStore { pub(super) async fn acquire_cross_object_lock( &self, + ) -> PersistenceResult { + self.acquire_mutation_lock(CROSS_OBJECT_ADVISORY_LOCK_KEY) + .await + } + + pub(super) async fn acquire_mutation_lock( + &self, + key: i64, ) -> PersistenceResult { let mut connection = self.pool.acquire().await.map_err(|e| map_db_error(&e))?; connection.close_on_drop(); @@ -125,7 +133,7 @@ impl PostgresStore { .await .map_err(|e| map_db_error(&e))?; sqlx::query("SELECT pg_advisory_lock($1)") - .bind(CROSS_OBJECT_ADVISORY_LOCK_KEY) + .bind(key) .execute(&mut *connection) .await .map_err(|e| map_db_error(&e))?; diff --git a/crates/openshell-server/src/ssh_identity.rs b/crates/openshell-server/src/ssh_identity.rs new file mode 100644 index 0000000000..74e43c9ffd --- /dev/null +++ b/crates/openshell-server/src/ssh_identity.rs @@ -0,0 +1,599 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox-lifetime SSH identities. Public objects contain only fingerprints; +//! the credential driver owns private keys and this internal row owns handles. + +use std::collections::HashMap; +use std::sync::Arc; + +use openshell_core::jwt::{SandboxLaunchAuthentication, SecretSshHostKey}; +use openshell_core::proto::Sandbox; +use openshell_core::proto::datamodel::v1::{CredentialHandle, ObjectMeta, Provider}; +use openshell_core::{ObjectId as _, ObjectName as _, ObjectWorkspace as _}; +use prost::Message as _; +use russh::keys::{Algorithm, HashAlg, PrivateKey}; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; +use tonic::Status; + +use crate::credentials::CredentialRuntime; +use crate::persistence::{PersistenceError, Store}; + +const OBJECT_TYPE: &str = "sandbox_ssh_identity_v1"; +const CANDIDATE_TYPE: &str = "sandbox_ssh_identity_candidate_v1"; +const CREDENTIAL_KEY: &str = "SSH_HOST_PRIVATE_KEY"; + +#[derive(Clone)] +pub struct SshIdentityStore { + store: Arc, + credentials: CredentialRuntime, + mutation: Arc>, +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::jwt::{CredentialEpoch, SecretJwt, SessionRotation, SupervisorAuthBundle}; + use openshell_core::{SandboxSessionId, sandbox_generation::SandboxGenerationId}; + + async fn fixture() -> SshIdentityStore { + let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); + let config = crate::Config::new(None) + .with_credential_drivers(["test-static"]) + .with_default_credential_driver(Some("test-static")); + let credentials = + CredentialRuntime::from_config_with_store(&config, store.clone()).unwrap(); + store.put_message(&sandbox("sandbox-a")).await.unwrap(); + SshIdentityStore::new(store, credentials) + } + + fn sandbox(id: &str) -> Sandbox { + Sandbox { + metadata: Some(ObjectMeta { + id: id.to_string(), + name: "same-name".to_string(), + workspace: "default".to_string(), + ..Default::default() + }), + ..Default::default() + } + } + + fn authentication() -> SandboxLaunchAuthentication { + SandboxLaunchAuthentication { + supervisor: SupervisorAuthBundle { + session_id: SandboxSessionId::new(), + runtime_generation: SandboxGenerationId::parse("generation").unwrap(), + session_rotation: SessionRotation::new(1).unwrap(), + auth_epoch: CredentialEpoch::new(1).unwrap(), + gateway_token: SecretJwt::parse("gateway-token").unwrap(), + gateway_expires_at: 0, + sandbox_token: SecretJwt::parse("sandbox-token").unwrap(), + sandbox_expires_at: 0, + ssh_host_private_key: None, + }, + gateway_id: "gateway".to_string(), + verification_keys: Vec::new(), + } + } + + #[tokio::test] + async fn concurrent_candidates_keep_one_resolvable_identity() { + let identities = fixture().await; + let sb = sandbox("sandbox-a"); + let (entered, release) = identities.credentials.gate_next_store(); + let first = identities.clone(); + let sb1 = sb.clone(); + let task = tokio::spawn(async move { first.get_or_create(&sb1).await.unwrap() }); + entered.await.unwrap(); + let winner = identities.get_or_create(&sb).await.unwrap(); + release.send(()).unwrap(); + let loser = task.await.unwrap(); + assert_eq!(winner.fingerprint, loser.fingerprint); + assert_eq!(identities.credentials.stored_credential_count(), Some(1)); + identities + .prepare(&mut sb.clone(), &mut authentication()) + .await + .unwrap(); + } + + #[tokio::test] + async fn identity_survives_reload_and_is_never_public() { + let identities = fixture().await; + let mut sb = sandbox("sandbox-a"); + let mut first = authentication(); + identities.prepare(&mut sb, &mut first).await.unwrap(); + let restored = + SshIdentityStore::new(identities.store.clone(), identities.credentials.clone()); + let mut next = authentication(); + restored.prepare(&mut sb, &mut next).await.unwrap(); + let private = first.supervisor.ssh_host_private_key.unwrap(); + assert_eq!( + private.expose_secret(), + next.supervisor + .ssh_host_private_key + .as_ref() + .unwrap() + .expose_secret() + ); + let key = PrivateKey::from_openssh(private.expose_secret()).unwrap(); + assert_eq!( + sb.host_key_fingerprint, + key.public_key().fingerprint(HashAlg::Sha256).to_string() + ); + assert!(!format!("{sb:?}").contains("PRIVATE KEY")); + assert!(!format!("{next:?}").contains("PRIVATE KEY")); + let stored = identities + .store + .get(OBJECT_TYPE, &SshIdentityStore::identity_id(&sb)) + .await + .unwrap() + .unwrap(); + assert!(!String::from_utf8_lossy(&stored.payload).contains("PRIVATE KEY")); + } + + #[tokio::test] + async fn deletion_retries_before_releasing_key_ownership() { + let identities = fixture().await; + let sb = sandbox("sandbox-a"); + let original = identities.get_or_create(&sb).await.unwrap(); + identities.credentials.fail_next_delete(); + assert!(identities.delete(&sb).await.is_err()); + assert_eq!( + identities.load(&sb).await.unwrap().unwrap().fingerprint, + original.fingerprint + ); + identities.delete(&sb).await.unwrap(); + assert!(identities.load(&sb).await.unwrap().is_none()); + assert_eq!(identities.credentials.stored_credential_count(), Some(0)); + identities.delete(&sb).await.unwrap(); + let replacement = identities + .get_or_create(&sandbox("sandbox-b")) + .await + .unwrap(); + assert_ne!(original.fingerprint, replacement.fingerprint); + } + + #[tokio::test] + async fn cancelled_prepare_finishes_before_deletion_and_cannot_recreate_identity() { + let identities = fixture().await; + let (entered, release) = identities.credentials.gate_next_store(); + let preparing = identities.clone(); + let task = tokio::spawn(async move { + preparing + .prepare(&mut sandbox("sandbox-a"), &mut authentication()) + .await + }); + entered.await.unwrap(); + task.abort(); + let deleting = identities + .store + .update_message_cas::("sandbox-a", 0, |sb| { + sb.set_phase(openshell_core::proto::SandboxPhase::Deleting.into()); + }) + .await + .unwrap(); + let cleanup = identities.clone(); + let deleted = tokio::spawn(async move { cleanup.delete(&deleting).await }); + release.send(()).unwrap(); + deleted.await.unwrap().unwrap(); + assert_eq!(identities.credentials.stored_credential_count(), Some(0)); + assert!( + identities + .prepare(&mut sandbox("sandbox-a"), &mut authentication()) + .await + .is_err() + ); + identities + .store + .delete("sandbox", "sandbox-a") + .await + .unwrap(); + assert!( + identities + .prepare(&mut sandbox("sandbox-a"), &mut authentication()) + .await + .is_err() + ); + assert_eq!(identities.credentials.stored_credential_count(), Some(0)); + } + + #[tokio::test] + async fn losing_candidate_cleanup_failure_remains_owned_for_deletion() { + let identities = fixture().await; + let sb = sandbox("sandbox-a"); + let (entered, release) = identities.credentials.gate_next_store(); + let candidate = identities.clone(); + let task = + tokio::spawn(async move { candidate.get_or_create(&sandbox("sandbox-a")).await }); + entered.await.unwrap(); + identities.get_or_create(&sb).await.unwrap(); + identities.credentials.fail_next_delete(); + release.send(()).unwrap(); + assert!(task.await.unwrap().is_err()); + assert_eq!(identities.credentials.stored_credential_count(), Some(2)); + identities.delete(&sb).await.unwrap(); + assert_eq!(identities.credentials.stored_credential_count(), Some(0)); + assert!( + identities + .store + .list_by_scope(CANDIDATE_TYPE, "sandbox-a", 100, 0) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn published_identity_is_never_replaced_when_storage_is_inconsistent() { + let identities = fixture().await; + let mut sb = sandbox("sandbox-a"); + identities + .prepare(&mut sb, &mut authentication()) + .await + .unwrap(); + identities + .store + .update_message_cas::("sandbox-a", 0, |sb| { + sb.host_key_fingerprint = "SHA256:inconsistent".to_string(); + }) + .await + .unwrap(); + assert!( + identities + .prepare(&mut sb, &mut authentication()) + .await + .is_err() + ); + let stored = identities + .store + .get_message::("sandbox-a") + .await + .unwrap() + .unwrap(); + assert_eq!(stored.host_key_fingerprint, "SHA256:inconsistent"); + identities + .store + .delete(OBJECT_TYPE, &SshIdentityStore::identity_id(&sb)) + .await + .unwrap(); + assert!( + identities + .prepare(&mut sb, &mut authentication()) + .await + .is_err() + ); + assert_eq!(identities.credentials.stored_credential_count(), Some(1)); + } + + #[tokio::test] + async fn default_credential_store_persists_key_across_runtime_reconstruction() { + let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); + let config = crate::Config::new(None); + let first = SshIdentityStore::new( + store.clone(), + CredentialRuntime::from_config_with_store(&config, store.clone()).unwrap(), + ); + let mut sb = sandbox("sandbox-a"); + first.store.put_message(&sb).await.unwrap(); + first.prepare(&mut sb, &mut authentication()).await.unwrap(); + let fingerprint = sb.host_key_fingerprint.clone(); + let next = SshIdentityStore::new( + store.clone(), + CredentialRuntime::from_config_with_store(&config, store).unwrap(), + ); + next.prepare(&mut sb, &mut authentication()).await.unwrap(); + assert_eq!(fingerprint, sb.host_key_fingerprint); + next.delete(&sb).await.unwrap(); + } +} + +#[derive(Serialize, Deserialize)] +struct StoredIdentity { + fingerprint: String, + // Protobuf bytes avoid introducing another durable handle schema. + credential_handle: Vec, +} + +impl SshIdentityStore { + pub(crate) fn new(store: Arc, credentials: CredentialRuntime) -> Self { + Self { + store, + credentials, + mutation: Arc::new(Mutex::new(())), + } + } + + /// Keep credential staging and publication alive if the calling RPC is + /// cancelled. The independent identity lock fences parent deletion on + /// every replica without nesting the creation cross-object lock. + pub(crate) async fn prepare( + &self, + sandbox: &mut Sandbox, + authentication: &mut SandboxLaunchAuthentication, + ) -> Result<(), Status> { + let identities = self.clone(); + let id = sandbox.object_id().to_string(); + let (updated, private_key) = tokio::spawn(async move { + let _local = identities.mutation.clone().lock_owned().await; + let _distributed = identities + .store + .acquire_ssh_identity_mutation_guard() + .await + .map_err(|_| Status::unavailable("lock sandbox SSH identity failed"))?; + let mut current = identities + .store + .get_message::(&id) + .await + .map_err(|_| Status::unavailable("load sandbox SSH identity owner failed"))? + .ok_or_else(|| Status::not_found("sandbox SSH identity owner was deleted"))?; + if current.phase() == i32::from(openshell_core::proto::SandboxPhase::Deleting) { + return Err(Status::failed_precondition( + "sandbox SSH identity owner is deleting", + )); + } + let private_key = identities.prepare_locked(&mut current).await?; + Ok::<_, Status>((current, private_key)) + }) + .await + .map_err(|_| Status::internal("sandbox SSH identity worker failed"))??; + *sandbox = updated; + authentication.supervisor.ssh_host_private_key = Some(private_key); + Ok(()) + } + + async fn prepare_locked(&self, sandbox: &mut Sandbox) -> Result { + let identity = self.get_or_create(sandbox).await?; + if !sandbox.host_key_fingerprint.is_empty() + && sandbox.host_key_fingerprint != identity.fingerprint + { + return Err(Status::internal( + "sandbox SSH identity does not match its published fingerprint", + )); + } + let handle = CredentialHandle::decode(identity.credential_handle.as_slice()) + .map_err(|_| Status::internal("invalid SSH identity credential handle"))?; + let provider = Self::owner( + sandbox, + HashMap::from([(CREDENTIAL_KEY.to_string(), handle)]), + ); + let mut resolved = self + .credentials + .resolve_provider_handles(&provider, 0) + .await?; + let private_key = resolved + .values + .remove(CREDENTIAL_KEY) + .ok_or_else(|| Status::unavailable("sandbox SSH host key is unavailable"))?; + let key = PrivateKey::from_openssh(&private_key) + .map_err(|_| Status::internal("invalid stored sandbox SSH host key"))?; + if key.public_key().fingerprint(HashAlg::Sha256).to_string() != identity.fingerprint { + return Err(Status::internal( + "sandbox SSH identity does not match its stored key", + )); + } + if sandbox.host_key_fingerprint.is_empty() { + let fingerprint = identity.fingerprint; + *sandbox = self + .store + .update_message_cas::(sandbox.object_id(), 0, |sandbox| { + sandbox.host_key_fingerprint.clone_from(&fingerprint); + }) + .await + .map_err(|_| Status::aborted("publish sandbox SSH identity failed"))?; + } + Ok(SecretSshHostKey::new(private_key)) + } + + fn identity_id(sandbox: &Sandbox) -> String { + format!("sandbox-ssh:{}", sandbox.object_id()) + } + + fn owner(sandbox: &Sandbox, handles: HashMap) -> Provider { + Provider { + metadata: Some(ObjectMeta { + id: sandbox.object_id().to_string(), + name: format!("sandbox-ssh-{}", sandbox.object_id()), + workspace: sandbox.object_workspace().to_string(), + ..Default::default() + }), + credential_handles: handles, + ..Default::default() + } + } + + async fn load(&self, sandbox: &Sandbox) -> Result, Status> { + self.store + .get(OBJECT_TYPE, &Self::identity_id(sandbox)) + .await + .map_err(|_| Status::unavailable("load sandbox SSH identity failed"))? + .map(|record| { + serde_json::from_slice(&record.payload) + .map_err(|_| Status::internal("invalid stored sandbox SSH identity")) + }) + .transpose() + } + + async fn get_or_create(&self, sandbox: &Sandbox) -> Result { + if let Some(identity) = self.load(sandbox).await? { + return Ok(identity); + } + if !sandbox.host_key_fingerprint.is_empty() { + return Err(Status::unavailable( + "sandbox SSH host key is missing; refusing to replace its identity", + )); + } + let key = PrivateKey::random(&mut rand_ssh::rng(), Algorithm::Ed25519) + .map_err(|_| Status::internal("generate sandbox SSH host key failed"))?; + let private_key = key + .to_openssh(russh::keys::ssh_key::LineEnding::default()) + .map_err(|_| Status::internal("encode sandbox SSH host key failed"))?; + let owner = Self::owner(sandbox, HashMap::new()); + // A distinct candidate path is essential for K8s/Vault: losing a CAS + // must never overwrite or delete the winning candidate's credential. + let candidate_id = uuid::Uuid::new_v4().to_string(); + let handles = self + .credentials + .store_provider_credentials_with_object_id( + owner.object_name(), + sandbox.object_workspace(), + sandbox.object_id(), + &candidate_id, + &HashMap::from([(CREDENTIAL_KEY.to_string(), private_key.to_string())]), + &HashMap::new(), + ) + .await?; + let handle = handles + .get(CREDENTIAL_KEY) + .ok_or_else(|| Status::internal("credential driver omitted sandbox SSH host key"))?; + let identity = StoredIdentity { + fingerprint: key.public_key().fingerprint(HashAlg::Sha256).to_string(), + credential_handle: handle.encode_to_vec(), + }; + let payload = serde_json::to_vec(&identity) + .map_err(|_| Status::internal("encode sandbox SSH identity failed"))?; + // Retain losing handles until cleanup succeeds, so deletion can retry + // a credential-driver failure instead of losing ownership of the key. + if self + .store + .create_scoped( + CANDIDATE_TYPE, + &candidate_id, + &candidate_id, + sandbox.object_workspace(), + sandbox.object_id(), + &payload, + None, + ) + .await + .is_err() + { + self.credentials + .delete_provider_credential_handles( + owner.object_name(), + sandbox.object_workspace(), + sandbox.object_id(), + &handles, + ) + .await?; + return Err(Status::unavailable("stage sandbox SSH identity failed")); + } + match self + .store + .create_scoped( + OBJECT_TYPE, + &Self::identity_id(sandbox), + sandbox.object_id(), + sandbox.object_workspace(), + sandbox.object_id(), + &payload, + None, + ) + .await + { + Ok(_) => { + self.store + .delete(CANDIDATE_TYPE, &candidate_id) + .await + .map_err(|_| { + Status::unavailable("finish sandbox SSH identity staging failed") + })?; + Ok(identity) + } + Err(error) => { + self.credentials + .delete_provider_credential_handles( + owner.object_name(), + sandbox.object_workspace(), + sandbox.object_id(), + &handles, + ) + .await?; + self.store + .delete(CANDIDATE_TYPE, &candidate_id) + .await + .map_err(|_| { + Status::unavailable("finish sandbox SSH identity cleanup failed") + })?; + if matches!(error, PersistenceError::UniqueViolation { .. }) { + self.load(sandbox).await?.ok_or_else(|| { + Status::aborted("sandbox SSH identity was concurrently removed") + }) + } else { + Err(Status::unavailable("persist sandbox SSH identity failed")) + } + } + } + } + + /// Credential deletion precedes row deletion so failure retains ownership + /// and can be retried by the normal sandbox deletion controller. + pub(crate) async fn delete(&self, sandbox: &Sandbox) -> Result<(), Status> { + let identities = self.clone(); + let sandbox = sandbox.clone(); + tokio::spawn(async move { + let _local = identities.mutation.clone().lock_owned().await; + let _distributed = identities + .store + .acquire_ssh_identity_mutation_guard() + .await + .map_err(|_| Status::unavailable("lock sandbox SSH identity failed"))?; + identities.delete_locked(&sandbox).await + }) + .await + .map_err(|_| Status::internal("sandbox SSH identity deletion worker failed"))? + } + + async fn delete_locked(&self, sandbox: &Sandbox) -> Result<(), Status> { + loop { + let candidates = self + .store + .list_by_scope(CANDIDATE_TYPE, sandbox.object_id(), 100, 0) + .await + .map_err(|_| Status::unavailable("load staged sandbox SSH identities failed"))?; + if candidates.is_empty() { + break; + } + for candidate in candidates { + let identity: StoredIdentity = serde_json::from_slice(&candidate.payload) + .map_err(|_| Status::internal("invalid staged sandbox SSH identity"))?; + self.delete_credential(sandbox, &identity).await?; + self.store + .delete(CANDIDATE_TYPE, &candidate.id) + .await + .map_err(|_| { + Status::unavailable("delete staged sandbox SSH identity failed") + })?; + } + } + let Some(identity) = self.load(sandbox).await? else { + return Ok(()); + }; + self.delete_credential(sandbox, &identity).await?; + self.store + .delete(OBJECT_TYPE, &Self::identity_id(sandbox)) + .await + .map_err(|_| Status::unavailable("delete sandbox SSH identity failed"))?; + Ok(()) + } + + async fn delete_credential( + &self, + sandbox: &Sandbox, + identity: &StoredIdentity, + ) -> Result<(), Status> { + let handle = CredentialHandle::decode(identity.credential_handle.as_slice()) + .map_err(|_| Status::internal("invalid SSH identity credential handle"))?; + let owner = Self::owner(sandbox, HashMap::new()); + self.credentials + .delete_provider_credential_handles( + owner.object_name(), + sandbox.object_workspace(), + sandbox.object_id(), + &HashMap::from([(CREDENTIAL_KEY.to_string(), handle)]), + ) + .await?; + Ok(()) + } +} diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index e6d8730aa0..36a12ab281 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -128,10 +128,12 @@ mod tests { // the additive file map while retaining the rest of the response. // Service authorization also extends both schemas additively. Legacy // payloads retain the safe Strip default. + // Sandbox field 6 adds its public SSH fingerprint. Older persisted + // sandboxes decode with an empty fingerprint, provisioned at next launch. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "2e156c6ad3c8eb51bcd30dc13b173fe339b38207a1b1f98f7be2e0cad8e3bd45"; + "354adc0a5945a9105b44602923bfdb8405766e91291fc651da5a9af968e43987"; const DURABLE_SCHEMA_SHA256: &str = - "38165d9d76f49fcfe98a12f241e032838a2376c1d1a87ea2796fd33b9b1a3541"; + "c2db375dc0b0ec51e6ef6ad6715704f3bec9cd11ccb05ced254a2c4efcbabe73"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = "761dea31a521b0650840fe2a823ad6e36a265ed323ba4506889781d630df0ee3"; // A persisted Sandbox without endpoint status retains its lifecycle fields; diff --git a/crates/openshell-supervisor-process/Cargo.toml b/crates/openshell-supervisor-process/Cargo.toml index 39d48a96e5..e8baa5a4b1 100644 --- a/crates/openshell-supervisor-process/Cargo.toml +++ b/crates/openshell-supervisor-process/Cargo.toml @@ -11,7 +11,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -openshell-core = { path = "../openshell-core" } +openshell-core = { path = "../openshell-core", features = ["jwt"] } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 87101f62a2..3bc4638f07 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -87,6 +87,7 @@ pub async fn start_boundary_access( port_forward: Arc, agent: Arc, supervisor_session_updates: Option>>, + host_key: Option, ) -> Result { let instance_id = uuid::Uuid::new_v4().to_string(); let terminating = Arc::new(AtomicBool::new(false)); @@ -106,6 +107,7 @@ pub async fn start_boundary_access( .await .map_err(|error| miette::miette!(error.to_string()))?; let main_session = crate::main_session::MainSession::from_boundary(attachment, agent); + let host_key = host_key.ok_or_else(|| miette::miette!("sandbox SSH host key is missing"))?; let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); let listen_path = ssh_socket_path.clone(); @@ -120,6 +122,7 @@ pub async fn start_boundary_access( ssh_port_forward, boundary_exec, Some(ssh_main_session), + host_key, ) .await { diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 40e0db1668..d3b95e9721 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -67,14 +67,27 @@ type SshServerInit = ( Option>, ); +/// A managed SSH server must never silently replace its pinned identity. +pub fn parse_host_key(key: Option<&openshell_core::jwt::SecretSshHostKey>) -> Result { + let key = key.ok_or_else(|| { + miette::miette!("sandbox SSH host key is missing; restart with a matching gateway release") + })?; + let key = PrivateKey::from_openssh(key.expose_secret()) + .map_err(|_| miette::miette!("invalid sandbox SSH host key"))?; + if key.algorithm() != Algorithm::Ed25519 || key.is_encrypted() { + return Err(miette::miette!( + "sandbox SSH host key must be an unencrypted Ed25519 key" + )); + } + Ok(key) +} + fn ssh_server_init( listen_path: &Path, ca_file_paths: &Option<(PathBuf, PathBuf)>, shared_socket: bool, + host_key: PrivateKey, ) -> Result { - let mut rng = rand::rng(); - let host_key = PrivateKey::random(&mut rng, Algorithm::Ed25519).into_diagnostic()?; - let mut config = russh::server::Config { server_id: russh::SshId::Standard(Cow::Owned(format!("SSH-2.0-OpenShell_{VERSION}"))), auth_rejection_time: Duration::from_secs(1), @@ -140,9 +153,10 @@ pub async fn run_ssh_server( port_forward: Arc, boundary_exec: Arc, main_session: Option>, + host_key: PrivateKey, ) -> Result<()> { let (listener, config, _ca_paths) = - match ssh_server_init(&listen_path, &ca_file_paths, shared_socket) { + match ssh_server_init(&listen_path, &ca_file_paths, shared_socket, host_key) { Ok(v) => { // Signal that the SSH server has bound the socket and is ready to // accept connections. The parent task awaits this before spawning @@ -1274,6 +1288,106 @@ mod tests { pub(super) struct AcceptAnyServerKey; + struct PinnedServerKey(String); + + impl russh::client::Handler for PinnedServerKey { + type Error = russh::Error; + + async fn check_server_key( + &mut self, + key: &russh::keys::PublicKey, + ) -> Result { + Ok(key.fingerprint(russh::keys::HashAlg::Sha256).to_string() == self.0) + } + } + + #[test] + fn managed_host_key_rejects_missing_and_invalid_material() { + assert!(parse_host_key(None).is_err()); + let secret = openshell_core::jwt::SecretSshHostKey::new("secret-invalid-key".to_string()); + let error = parse_host_key(Some(&secret)).unwrap_err().to_string(); + assert!(!error.contains(secret.expose_secret())); + let key = PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(); + let secret = openshell_core::jwt::SecretSshHostKey::new( + key.to_openssh(russh::keys::ssh_key::LineEnding::default()) + .unwrap() + .to_string(), + ); + let decoded = parse_host_key(Some(&secret)).unwrap(); + assert_eq!(key.public_key(), decoded.public_key()); + } + + #[cfg(unix)] + #[tokio::test] + async fn pinned_identity_matches_direct_and_relayed_ssh_after_listener_restart() { + let key = PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(); + let fingerprint = key + .public_key() + .fingerprint(russh::keys::HashAlg::Sha256) + .to_string(); + let temp = tempfile::tempdir().unwrap(); + for relay in [false, true] { + let socket = temp.path().join("ssh.sock"); + let (listener, config, _) = + ssh_server_init(&socket, &None, false, key.clone()).unwrap(); + let server = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let config = config.clone(); + tokio::spawn(async move { + let handler = SshHandler::new( + Arc::new(TestLoopbackConnector), + Arc::new(RejectingExec), + None, + ); + if let Ok(session) = + russh::server::run_stream(config, stream, handler).await + { + let _ = session.await; + } + }); + } + }); + let direct = tokio::net::UnixStream::connect(&socket).await.unwrap(); + let stream: openshell_isolation_interface::contract::BoundaryDuplexStream = if relay { + let (client, mut tunnel) = tokio::io::duplex(64 * 1024); + tokio::spawn(async move { + let mut direct = direct; + let _ = tokio::io::copy_bidirectional(&mut tunnel, &mut direct).await; + }); + Box::new(client) + } else { + Box::new(direct) + }; + let mut client = russh::client::connect_stream( + Arc::new(russh::client::Config::default()), + stream, + PinnedServerKey(fingerprint.clone()), + ) + .await + .unwrap(); + assert!(matches!( + client.authenticate_none("sandbox").await.unwrap(), + russh::client::AuthResult::Success + )); + client + .disconnect(russh::Disconnect::ByApplication, "done", "") + .await + .unwrap(); + let stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); + assert!( + russh::client::connect_stream( + Arc::new(russh::client::Config::default()), + stream, + PinnedServerKey("SHA256:wrong".to_string()) + ) + .await + .is_err() + ); + server.abort(); + let _ = server.await; + } + } + impl russh::client::Handler for AcceptAnyServerKey { type Error = russh::Error; @@ -1814,7 +1928,13 @@ mod tests { set_file_mode(&parent, 0o775); let socket = parent.join("ssh.sock"); - let (listener, _, _) = ssh_server_init(&socket, &None, false).unwrap(); + let (listener, _, _) = ssh_server_init( + &socket, + &None, + false, + PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(), + ) + .unwrap(); drop(listener); assert_eq!(file_mode(&parent), 0o700); @@ -1830,7 +1950,13 @@ mod tests { set_file_mode(&parent, 0o775); let socket = parent.join("ssh.sock"); - let (listener, _, _) = ssh_server_init(&socket, &None, true).unwrap(); + let (listener, _, _) = ssh_server_init( + &socket, + &None, + true, + PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(), + ) + .unwrap(); drop(listener); assert_eq!(file_mode(&parent), 0o775); @@ -1841,7 +1967,13 @@ mod tests { #[tokio::test] async fn ssh_server_abstract_socket_cannot_be_replaced_while_bound() { let socket = PathBuf::from(format!("@openshell-ssh-test-{}", uuid::Uuid::new_v4())); - let (listener, _, _) = ssh_server_init(&socket, &None, true).unwrap(); + let (listener, _, _) = ssh_server_init( + &socket, + &None, + true, + PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(), + ) + .unwrap(); assert!( !socket.exists(), diff --git a/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs b/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs index f326c63cf2..3d58308562 100644 --- a/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs +++ b/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs @@ -111,7 +111,13 @@ impl Connection { async fn with_window(main: Arc, window: Option) -> Self { let dir = tempfile::tempdir().unwrap(); let socket = dir.path().join("ssh.sock"); - let (listener, mut config, _) = ssh_server_init(&socket, &None, false).unwrap(); + let (listener, mut config, _) = ssh_server_init( + &socket, + &None, + false, + PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(), + ) + .unwrap(); // Preserve whether production enables probes. Only shorten the interval. let config_mut = Arc::get_mut(&mut config).unwrap(); config_mut.keepalive_interval = config_mut.keepalive_interval.map(|_| TEST_INTERVAL); @@ -407,7 +413,13 @@ async fn waiting_writer_eof_cancels_acquisition() { #[tokio::test] async fn production_config_probes_before_the_receive_deadline() { let dir = tempfile::tempdir().unwrap(); - let (_, config, _) = ssh_server_init(&dir.path().join("ssh.sock"), &None, false).unwrap(); + let (_, config, _) = ssh_server_init( + &dir.path().join("ssh.sock"), + &None, + false, + PrivateKey::random(&mut rand::rng(), Algorithm::Ed25519).unwrap(), + ) + .unwrap(); assert_eq!(config.keepalive_interval, Some(Duration::from_secs(15))); assert_eq!(config.keepalive_max, 3); assert_eq!(SSH_PEER_TIMEOUT, Duration::from_mins(1)); diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 63c5302673..9b8cab8ead 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -683,6 +683,13 @@ pub async fn run_sandbox( )); } let sandbox_bearer = openshell_core::grpc_client::install_supervisor_auth_bundle(&auth_bundle)?; + let ssh_host_key = if ssh_socket_path.is_some() { + Some(openshell_supervisor_process::ssh::parse_host_key( + auth_bundle.ssh_host_private_key.as_ref(), + )?) + } else { + None + }; let (image_yaml, invalid_image) = openshell_sandbox_backend::OpenShellRuntimeBackend::discover_policy( runtime_descriptor.clone(), @@ -1168,6 +1175,7 @@ pub async fn run_sandbox( running.loopback_connector(), agent.clone(), Some(supervisor_session_updates), + ssh_host_key, ) .await?; info!(backend = %backend_name, "Control-mode access plane started"); diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 8194b0df5b..23ecbb29cf 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -799,6 +799,29 @@ openshell sandbox create --from registry.example.com/your-org/claude-agent:lates ## SSH Config +Each sandbox has its own SSH host key. The key stays the same when the +supervisor or container restarts, a pod is rescheduled, or the sandbox is +stopped and started. Deleting the sandbox removes its key. Creating another +sandbox with the same name creates a different key. + +The gateway returns the public fingerprint in `Sandbox.host_key_fingerprint` +from sandbox creation, lookup, and listing, and in +`CreateSshSessionResponse.host_key_fingerprint`. The value uses OpenSSH's +`SHA256:` format. Read it over an authenticated gateway connection, +pin it to the sandbox ID, and reject SSH connections that present a different +fingerprint. This also applies to clients that connect directly to the +supervisor's SSH socket. + +The gateway keeps the private key in its credential store and delivers it +only to the supervisor. The sandbox workload cannot read the key. Drivers, +gateway, and supervisor must use the matching OpenShell release. Older +runtimes can return an empty fingerprint; an empty value cannot verify an +SSH server's identity. + +The CLI and TUI continue to use their existing SSH connection settings. +Exposing the fingerprint does not enable automatic host-key verification in +those clients. + Generate an SSH config entry for a sandbox so tools like VS Code Remote-SSH can connect directly: ```shell diff --git a/e2e/rust/tests/ssh_host_identity.rs b/e2e/rust/tests/ssh_host_identity.rs new file mode 100644 index 0000000000..6872b898a0 --- /dev/null +++ b/e2e/rust/tests/ssh_host_identity.rs @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e")] + +use std::path::Path; +use std::time::Duration; + +use openshell_e2e::harness::cli::{ + run_cli, wait_for_healthy, wait_for_sandbox_exec_contains, wait_for_sandbox_phase, +}; +use openshell_e2e::harness::gateway::ManagedGateway; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::sandbox::SandboxGuard; + +async fn fingerprint(name: &str) -> String { + let (output, code) = run_cli(&["sandbox", "get", name, "--output", "json"]).await; + assert_eq!(code, 0, "sandbox get failed: {output}"); + let sandbox: serde_json::Value = serde_json::from_str(&strip_ansi(&output)).unwrap(); + let fingerprint = sandbox["host_key_fingerprint"] + .as_str() + .unwrap() + .to_string(); + assert!(fingerprint.starts_with("SHA256:")); + let (output, code) = run_cli(&["sandbox", "list", "--output", "json"]).await; + assert_eq!(code, 0, "sandbox list failed: {output}"); + let list: serde_json::Value = serde_json::from_str(&strip_ansi(&output)).unwrap(); + let listed = list["sandboxes"] + .as_array() + .unwrap() + .iter() + .find(|sandbox| sandbox["name"] == name) + .unwrap(); + assert_eq!(listed["host_key_fingerprint"], fingerprint); + fingerprint +} + +async fn pinned_ssh(config: &Path, known_hosts: &Path, name: &str, first: bool) { + let output = tokio::time::timeout( + Duration::from_secs(30), + tokio::process::Command::new("ssh") + .kill_on_drop(true) + .arg("-F") + .arg(config) + .arg("-o") + .arg(if first { + "StrictHostKeyChecking=accept-new" + } else { + "StrictHostKeyChecking=yes" + }) + .arg("-o") + .arg(format!("UserKnownHostsFile={}", known_hosts.display())) + .arg("-o") + .arg("BatchMode=yes") + .arg(format!("openshell-{name}.default")) + .args(["printf", "ssh-identity-ok"]) + .output(), + ) + .await + .unwrap() + .unwrap(); + assert!( + output.status.success(), + "pinned SSH failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout), "ssh-identity-ok"); +} + +#[tokio::test] +async fn ssh_host_identity_survives_restarts_and_changes_after_recreation() { + let mut sandbox = SandboxGuard::create(&[]).await.unwrap(); + let name = sandbox.name.clone(); + let original = fingerprint(&name).await; + let directory = tempfile::tempdir().unwrap(); + let config = directory.path().join("ssh_config"); + let known_hosts = directory.path().join("known_hosts"); + let (output, code) = run_cli(&["sandbox", "ssh-config", &name]).await; + assert_eq!(code, 0, "SSH config failed: {output}"); + std::fs::write(&config, output).unwrap(); + pinned_ssh(&config, &known_hosts, &name, true).await; + let key = tokio::process::Command::new("ssh-keygen") + .args(["-l", "-E", "sha256", "-f"]) + .arg(&known_hosts) + .output() + .await + .unwrap(); + assert!(key.status.success()); + assert!( + String::from_utf8_lossy(&key.stdout) + .split_whitespace() + .any(|word| word == original) + ); + sandbox + .exec(&["sh", "-c", "test ! -r /.openshell/supervisor/auth.json"]) + .await + .unwrap(); + + let (output, code) = run_cli(&["sandbox", "stop", &name]).await; + assert_eq!(code, 0, "stop failed: {output}"); + wait_for_sandbox_phase(&name, "Stopped", Duration::from_secs(60)) + .await + .unwrap(); + assert_eq!(fingerprint(&name).await, original); + let (output, code) = run_cli(&["sandbox", "start", &name]).await; + assert_eq!(code, 0, "start failed: {output}"); + wait_for_sandbox_phase(&name, "Ready", Duration::from_secs(120)) + .await + .unwrap(); + assert_eq!(fingerprint(&name).await, original); + pinned_ssh(&config, &known_hosts, &name, false).await; + + if let Some(gateway) = ManagedGateway::from_env().unwrap() { + gateway.stop().unwrap(); + gateway.start().unwrap(); + wait_for_healthy(Duration::from_secs(120)).await.unwrap(); + wait_for_sandbox_phase(&name, "Ready", Duration::from_secs(120)) + .await + .unwrap(); + wait_for_sandbox_exec_contains( + &name, + &["printf", "ssh-identity-ready"], + "ssh-identity-ready", + Duration::from_secs(240), + ) + .await + .unwrap(); + assert_eq!(fingerprint(&name).await, original); + pinned_ssh(&config, &known_hosts, &name, false).await; + } else { + eprintln!("Skipping gateway restart check: this run uses an existing gateway"); + } + + sandbox.cleanup().await; + let mut replacement = SandboxGuard::create(&["--name", &name]).await.unwrap(); + assert_ne!(fingerprint(&name).await, original); + replacement.cleanup().await; +} diff --git a/proto/openshell.proto b/proto/openshell.proto index 83ffbe75ad..a62e7ae48b 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -1006,6 +1006,10 @@ message Sandbox { SandboxSpec spec = 2; // Latest user-facing observed status derived by the gateway. SandboxStatus status = 3; + // Read-only SSH host identity in OpenSSH SHA256: form. Stable for + // this sandbox's lifetime, including runtime restarts. Empty on legacy + // runtimes that do not provide a gateway-managed SSH identity. + string host_key_fingerprint = 6; // Read-only provenance for sandboxes created from a reusable workload template. SandboxWorkloadTemplateProvenance created_from_workload_template = 20; @@ -1753,7 +1757,8 @@ message CreateSshSessionResponse { // Gateway scheme. Must be exactly "http" or "https". string gateway_scheme = 5; - // Optional host key fingerprint. If non-empty, [A-Za-z0-9:+/=-] only. + // Expected sandbox SSH host identity in OpenSSH SHA256: form. + // Stable for the sandbox's lifetime. Empty only for legacy runtimes. string host_key_fingerprint = 7; // Absolute expiry. Absence means no expiry. diff --git a/python/openshell/sandbox.py b/python/openshell/sandbox.py index 485af39d04..745c31e81a 100644 --- a/python/openshell/sandbox.py +++ b/python/openshell/sandbox.py @@ -521,6 +521,8 @@ class SandboxRef: service_urls: Mapping[str, str] = field( default_factory=_ImmutableLabels, compare=False ) + # Stable SSH host identity; absent on older runtimes. + host_key_fingerprint: str | None = None def __post_init__(self) -> None: object.__setattr__(self, "labels", _ImmutableLabels(self.labels)) @@ -1796,6 +1798,7 @@ def _sandbox_ref( labels=sandbox.metadata.labels if sandbox.metadata else {}, created_from_workload_template=provenance, service_urls=service_urls or {}, + host_key_fingerprint=sandbox.host_key_fingerprint or None, ) diff --git a/python/openshell/sandbox_test.py b/python/openshell/sandbox_test.py index 3b744451cb..40fc7c205f 100644 --- a/python/openshell/sandbox_test.py +++ b/python/openshell/sandbox_test.py @@ -2728,6 +2728,13 @@ def test_sandbox_ref_retains_gateway_labels() -> None: assert dict(ref.labels) == {"aiq": "deep-research", "env": "dev"} +def test_sandbox_ref_exposes_host_identity_and_accepts_legacy_response() -> None: + proto = _make_sandbox_proto("sandbox-1", "job-1") + assert _sandbox_ref(proto).host_key_fingerprint is None + proto.host_key_fingerprint = "SHA256:expected" + assert _sandbox_ref(proto).host_key_fingerprint == "SHA256:expected" + + def test_sandbox_ref_includes_main_process_result() -> None: proto = _make_sandbox_proto("sandbox-1", "job-1") proto.status.exit_code = 0 diff --git a/sdk/go/openshell/v1/internal/converter/sandbox.go b/sdk/go/openshell/v1/internal/converter/sandbox.go index 7d81a7c828..cacd18e964 100644 --- a/sdk/go/openshell/v1/internal/converter/sandbox.go +++ b/sdk/go/openshell/v1/internal/converter/sandbox.go @@ -21,7 +21,7 @@ func SandboxFromProto(s *pb.Sandbox) *types.Sandbox { return nil } - result := &types.Sandbox{} + result := &types.Sandbox{HostKeyFingerprint: s.GetHostKeyFingerprint()} if m := s.GetMetadata(); m != nil { result.ID = m.GetId() diff --git a/sdk/go/openshell/v1/internal/converter/sandbox_test.go b/sdk/go/openshell/v1/internal/converter/sandbox_test.go index 2099b5b65b..ada0c6ee2e 100644 --- a/sdk/go/openshell/v1/internal/converter/sandbox_test.go +++ b/sdk/go/openshell/v1/internal/converter/sandbox_test.go @@ -51,6 +51,7 @@ func TestSandboxFromProto(t *testing.T) { gpuCount := uint32(2) exitCode := int32(0) proto := &pb.Sandbox{ + HostKeyFingerprint: "SHA256:expected", Metadata: &dm.ObjectMeta{ Id: "sb-1", Name: "my-sandbox", @@ -119,6 +120,7 @@ func TestSandboxFromProto(t *testing.T) { } s := SandboxFromProto(proto) + assert.Equal(t, "SHA256:expected", s.HostKeyFingerprint) require.NotNil(t, s) assert.Equal(t, "sb-1", s.ID) diff --git a/sdk/go/openshell/v1/types/sandbox.go b/sdk/go/openshell/v1/types/sandbox.go index c8e7d43ef3..8e57c14f8e 100644 --- a/sdk/go/openshell/v1/types/sandbox.go +++ b/sdk/go/openshell/v1/types/sandbox.go @@ -7,6 +7,8 @@ import "time" // Sandbox represents a sandbox instance. type Sandbox struct { + // HostKeyFingerprint is the stable SSH identity; empty on older runtimes. + HostKeyFingerprint string ID string Name string CreatedAt time.Time diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 109d2ef454..af6c2b5d27 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -2270,6 +2270,10 @@ type Sandbox struct { Spec *SandboxSpec `protobuf:"bytes,2,opt,name=spec,proto3" json:"spec,omitempty"` // Latest user-facing observed status derived by the gateway. Status *SandboxStatus `protobuf:"bytes,3,opt,name=status,proto3" json:"status,omitempty"` + // Read-only SSH host identity in OpenSSH SHA256: form. Stable for + // this sandbox's lifetime, including runtime restarts. Empty on legacy + // runtimes that do not provide a gateway-managed SSH identity. + HostKeyFingerprint string `protobuf:"bytes,6,opt,name=host_key_fingerprint,json=hostKeyFingerprint,proto3" json:"host_key_fingerprint,omitempty"` // Read-only provenance for sandboxes created from a reusable workload template. CreatedFromWorkloadTemplate *SandboxWorkloadTemplateProvenance `protobuf:"bytes,20,opt,name=created_from_workload_template,json=createdFromWorkloadTemplate,proto3" json:"created_from_workload_template,omitempty"` unknownFields protoimpl.UnknownFields @@ -2327,6 +2331,13 @@ func (x *Sandbox) GetStatus() *SandboxStatus { return nil } +func (x *Sandbox) GetHostKeyFingerprint() string { + if x != nil { + return x.HostKeyFingerprint + } + return "" +} + func (x *Sandbox) GetCreatedFromWorkloadTemplate() *SandboxWorkloadTemplateProvenance { if x != nil { return x.CreatedFromWorkloadTemplate @@ -6060,7 +6071,8 @@ type CreateSshSessionResponse struct { GatewayPort uint32 `protobuf:"varint,4,opt,name=gateway_port,json=gatewayPort,proto3" json:"gateway_port,omitempty"` // Gateway scheme. Must be exactly "http" or "https". GatewayScheme string `protobuf:"bytes,5,opt,name=gateway_scheme,json=gatewayScheme,proto3" json:"gateway_scheme,omitempty"` - // Optional host key fingerprint. If non-empty, [A-Za-z0-9:+/=-] only. + // Expected sandbox SSH host identity in OpenSSH SHA256: form. + // Stable for the sandbox's lifetime. Empty only for legacy runtimes. HostKeyFingerprint string `protobuf:"bytes,7,opt,name=host_key_fingerprint,json=hostKeyFingerprint,proto3" json:"host_key_fingerprint,omitempty"` // Absolute expiry. Absence means no expiry. ExpirationTime *timestamppb.Timestamp `protobuf:"bytes,108,opt,name=expiration_time,json=expirationTime,proto3" json:"expiration_time,omitempty"` @@ -17890,11 +17902,12 @@ const file_openshell_proto_rawDesc = "" + "\x0flimit_supported\x18\x01 \x01(\bR\x0elimitSupported\"\x95\x01\n" + "\x17GpuResourceCapabilities\x12>\n" + "\x1bdefault_selection_supported\x18\x01 \x01(\bR\x19defaultSelectionSupported\x12:\n" + - "\x19count_selection_supported\x18\x02 \x01(\bR\x17countSelectionSupported\"\xce\x02\n" + + "\x19count_selection_supported\x18\x02 \x01(\bR\x17countSelectionSupported\"\x80\x03\n" + "\aSandbox\x12>\n" + "\bmetadata\x18\x01 \x01(\v2\".openshell.datamodel.v1.ObjectMetaR\bmetadata\x12-\n" + "\x04spec\x18\x02 \x01(\v2\x19.openshell.v1.SandboxSpecR\x04spec\x123\n" + - "\x06status\x18\x03 \x01(\v2\x1b.openshell.v1.SandboxStatusR\x06status\x12t\n" + + "\x06status\x18\x03 \x01(\v2\x1b.openshell.v1.SandboxStatusR\x06status\x120\n" + + "\x14host_key_fingerprint\x18\x06 \x01(\tR\x12hostKeyFingerprint\x12t\n" + "\x1ecreated_from_workload_template\x18\x14 \x01(\v2/.openshell.v1.SandboxWorkloadTemplateProvenanceR\x1bcreatedFromWorkloadTemplateJ\x04\b\x04\x10\x05J\x04\b\x05\x10\x06R\x05phaseR\x16current_policy_version\"\x8a\x05\n" + "\vSandboxSpec\x12\x1b\n" + "\tlog_level\x18\x01 \x01(\tR\blogLevel\x12L\n" + diff --git a/sdk/typescript/src/client.test.ts b/sdk/typescript/src/client.test.ts index 178a73cc6e..45464409e4 100644 --- a/sdk/typescript/src/client.test.ts +++ b/sdk/typescript/src/client.test.ts @@ -65,6 +65,24 @@ function readySandbox( const enc = (s: string) => new TextEncoder().encode(s); +describe('sandbox SSH host identity', () => { + it('exposes the fingerprint from get and list, and accepts legacy responses', async () => { + let fingerprint = 'SHA256:expected'; + const sandbox = client({ + getSandbox: () => ({ + sandbox: { metadata: { id: 'id', name: 'work', workspace: 'default' }, hostKeyFingerprint: fingerprint }, + }), + listSandboxes: () => ({ + sandboxes: [{ metadata: { id: 'id', name: 'work', workspace: 'default' }, hostKeyFingerprint: fingerprint }], + }), + }); + expect((await sandbox.get('work')).hostKeyFingerprint).toBe(fingerprint); + expect((await sandbox.list().all())[0]?.hostKeyFingerprint).toBe(fingerprint); + fingerprint = ''; + expect((await sandbox.get('work')).hostKeyFingerprint).toBeUndefined(); + }); +}); + describe('deletion outcomes', () => { it('defaults to strict deletion and preserves accepted identity and unknown values', async () => { const flags: boolean[] = []; diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.ts index e73bccd314..f735ff9495 100644 --- a/sdk/typescript/src/client.ts +++ b/sdk/typescript/src/client.ts @@ -216,6 +216,8 @@ export interface SandboxRef { labels: Record; /** u64 rendered as a string — JS numbers can't hold it safely. */ resourceVersion: string; + /** Stable SSH host identity; absent on older runtimes. */ + hostKeyFingerprint?: string; mainProcessInstanceId?: string; exitCode?: number; createdFromWorkloadTemplate?: SandboxWorkloadTemplateProvenance; @@ -535,6 +537,7 @@ function sandboxRef(sandbox: Sandbox | undefined, serviceUrls: Record`. For configured MCP-over-HTTP endpoints, JSON output exposes each address together with `last_result` and `last_reported_at` in `endpoint_statuses`. Select the endpoint by host, path, and ports, then check the reported failure boundary. `last_reported_at` records gateway acceptance time and can advance when retained evidence is accepted after a reset. Results do not expire or prove current availability; `HttpResponseReceived` can still contain a tool error. If several paths share a host and port, a failure before the path is known remains in logs. Verify the actual operation when current tool availability matters. - On Docker Desktop, repeated `Policy fetch failed after 5 attempts` messages can mean host networking is disabled. Enable host networking in Docker diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index 3977018a01..08706cc985 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -490,6 +490,13 @@ stopped or completed. Starting a retained `Completed` or invalidates SSH sessions from the previous runtime generation. Delete remains the operation that removes retained state. +The sandbox's SSH host identity survives stop/start and runtime restarts. +API clients can read its public fingerprint from the sandbox resource or +SSH-session response and pin it to the sandbox ID. Recreating a deleted +sandbox, even under the same name, creates a different identity. See the +[sandbox SSH documentation](https://docs.nvidia.com/openshell/latest/how-it-works/sandboxes/overview.md) +for verification behavior and release compatibility. + --- ## Workflow 4: Policy Iteration Loop