diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 71fb790b11..dee215e447 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -58,11 +58,22 @@ pub const CONDITION_WORKSPACE_VALIDATION_FAILED: &str = "WorkspaceValidationFail /// Supervisor exit status reserved for OCI workspace validation failures. /// -/// Local container drivers translate this status into +/// Local container drivers translate a supervisor exit with this status into /// [`CONDITION_WORKSPACE_VALIDATION_FAILED`] so users receive the specific /// provisioning failure rather than a generic container exit. pub const SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED: i32 = 78; +/// Error context for a rejected image-provided OCI working directory. +/// +/// The supervisor recognizes it in a failed agent start and exits with +/// [`SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED`]. +pub const WORKSPACE_VALIDATION_ERROR_CONTEXT: &str = "image workspace validation failed"; + +/// Driver condition message for [`CONDITION_WORKSPACE_VALIDATION_FAILED`]. +/// Fixed text, because supervisor output may contain secrets. +pub const WORKSPACE_VALIDATION_FAILED_MESSAGE: &str = + "OCI WorkingDir is not usable by the sandbox identity"; + /// Ready-condition reason when a container was terminated by an external signal. /// /// SIGKILL/SIGTERM (exit 137/143) is what a Podman/Docker machine or daemon diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 950b2742e8..ec176f6adc 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -26,10 +26,12 @@ use futures::{Stream, StreamExt}; use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, - LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, - SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, + CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_WORKSPACE_VALIDATION_FAILED, + LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, + LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SANDBOX_RUNTIME_IMAGE_BINARY_PATH, + SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_FAILED_MESSAGE, + extract_first_tar_entry, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -1854,7 +1856,7 @@ impl DockerComputeDriver { .await; cleanup_docker_boundary_state(sandbox, &self.config); return Err(DockerProvisioningFailure::new( - "ControlSupervisorStartFailed", + supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"), status.message(), )); } @@ -2146,7 +2148,7 @@ impl DockerComputeDriver { Err(status) => { handle_docker_runtime_failure( failure_context, - "ControlSupervisorExited", + supervisor_start_failure_reason(&status, "ControlSupervisorExited"), format!( "failed to start Docker control supervisor: {}", status.message() @@ -5272,9 +5274,11 @@ async fn spawn_docker_control_process( ).await; return; } + let mut reason = "ControlSupervisorExited"; let mut message = match result { Some(Ok(status)) => { warn!(%sandbox_id, status = status.status_code, "Docker supervisor container exited unexpectedly"); + reason = supervisor_exit_reason(status.status_code); format!("Docker supervisor container exited with status {}", status.status_code) } Some(Err(error)) => { @@ -5299,12 +5303,7 @@ async fn spawn_docker_control_process( if monitored_shutdown.load(Ordering::Acquire) { return; } - handle_docker_runtime_failure( - failure_context, - "ControlSupervisorExited", - message, - ) - .await; + handle_docker_runtime_failure(failure_context, reason, message).await; }, } }); @@ -5351,6 +5350,13 @@ async fn wait_for_docker_supervisor_ready( let state = inspected.state.unwrap_or_default(); match state.health.and_then(|health| health.status) { Some(HealthStatusEnum::HEALTHY) => return Ok(()), + _ if state.running == Some(false) + && state.exit_code + == Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)) => + { + let log_tail = docker_container_log_tail(docker, supervisor_id).await; + return Err(workspace_validation_status(&log_tail)); + } _ if state.running == Some(false) => { let log_tail = docker_container_log_tail(docker, supervisor_id).await; warn!(sandbox_id, supervisor_id, supervisor_logs = %log_tail, "Docker supervisor exited before becoming ready"); @@ -5364,6 +5370,38 @@ async fn wait_for_docker_supervisor_ready( } } +/// Startup failure for a supervisor that exited because the sandbox rejected +/// the image working directory. The fixed message prefix identifies it for +/// [`supervisor_start_failure_reason`]. +fn workspace_validation_status(log_tail: &str) -> Status { + Status::failed_precondition(format!( + "{WORKSPACE_VALIDATION_FAILED_MESSAGE}{}", + format_log_tail(log_tail) + )) +} + +/// Condition reason for a supervisor that failed before becoming ready. +fn supervisor_start_failure_reason(status: &Status, default: &'static str) -> &'static str { + if status.code() == tonic::Code::FailedPrecondition + && status + .message() + .starts_with(WORKSPACE_VALIDATION_FAILED_MESSAGE) + { + CONDITION_WORKSPACE_VALIDATION_FAILED + } else { + default + } +} + +/// Condition reason for a supervisor that exited with `status_code`. +fn supervisor_exit_reason(status_code: i64) -> &'static str { + if status_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) { + CONDITION_WORKSPACE_VALIDATION_FAILED + } else { + "ControlSupervisorExited" + } +} + fn format_log_tail(log_tail: &str) -> String { // gRPC status messages travel in HTTP/2 headers. A full 16 KiB container // tail can exceed the client's 16 KiB header budget and hide the real diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 08576f422a..c0ca749795 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -3619,6 +3619,28 @@ fn docker_oom_kill_stays_terminal_despite_137() { assert_eq!(ready_reason(&sandbox), CONDITION_EXITED); } +#[test] +fn supervisor_workspace_validation_exit_is_reported_explicitly() { + let status = workspace_validation_status("image workspace validation failed: denied"); + assert!(status.message().contains("log tail: image workspace")); + assert_eq!( + supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"), + CONDITION_WORKSPACE_VALIDATION_FAILED + ); + assert_eq!( + supervisor_start_failure_reason( + &Status::failed_precondition("provider SPIFFE socket has no parent directory"), + "ControlSupervisorStartFailed" + ), + "ControlSupervisorStartFailed" + ); + assert_eq!( + supervisor_exit_reason(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)), + CONDITION_WORKSPACE_VALIDATION_FAILED + ); + assert_eq!(supervisor_exit_reason(1), "ControlSupervisorExited"); +} + #[test] fn concurrent_container_removal_is_idempotent() { let removing = BollardError::DockerResponseServerError { diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 85a31f2cf8..e2efc5f3e7 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -44,7 +44,8 @@ only the channel bootstrap into the existing channel volume, preserving the workspace. The workload starts before the supervisor so its user namespace exists when the supervisor joins it; a stopped supervisor resolves that namespace again on its next start. The driver creates the managed workspace volume owned by -the workload's final UID and GID, so the workload never starts as root. +the workload's final UID and GID, so the workload never starts as root. Custom +image workspaces have no workspace volume or upload. The runtime must pass the sandbox's unprivileged enforcement probe, including nested seccomp notification and Landlock. Unsupported runtime defaults fail @@ -92,6 +93,20 @@ binary extraction path. `supervisor_image` supplies the dynamically linked glibc `/openshell-supervisor` binary outside the workload. Image and request environment belong to agent children, never the supervisor process. +## OCI working directory + +OpenShell reads `WORKDIR` from the workload image. If it is unset, `/`, or +`/sandbox`, OpenShell uses its managed `/sandbox` workspace volume. A custom +path must be absolute and normalized, and cannot overlap `/proc`, `/sys`, +`/dev`, OpenShell-reserved paths, or the workload's private control and CA +mounts. Image volumes and driver mounts cannot cover it; mounts nested below it +remain valid. + +A custom path stays in the image's container filesystem with its ownership and +permissions. The workload starts as the final non-root user, which must be able +to reach and write the directory. Agent commands use the path as their working +directory. + ## Lifecycle and readiness Create builds both stopped containers and stages the private archives before diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index f29ab1113e..6527643613 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -121,6 +121,10 @@ pub struct ContainerState { /// container-log marker. It is never deserialized from Podman. #[serde(skip)] pub startup_diagnostic: Option, + /// Exit status of the sandbox's supervisor companion when it stopped + /// before the workload. It is never deserialized from Podman. + #[serde(skip)] + pub supervisor_exit_code: Option, } #[derive(Debug, Clone, serde::Deserialize)] @@ -187,6 +191,10 @@ pub struct ImageConfig { pub user: String, #[serde(default)] pub env: Vec, + #[serde(default)] + pub working_dir: String, + #[serde(default)] + pub volumes: Option>, } /// Whether a driver-owned volume has exactly the options `OpenShell` creates it @@ -1222,12 +1230,12 @@ mod tests { } #[tokio::test] - async fn inspect_image_reads_immutable_id_and_oci_user() { + async fn inspect_image_reads_immutable_id_and_oci_config() { let (socket_path, request_log, handle) = spawn_podman_stub( "inspect-image", vec![StubResponse::new( StatusCode::OK, - r#"{"Id":"sha256:immutable","Config":{"User":"app:staff"}}"#, + r#"{"Id":"sha256:immutable","Config":{"User":"app:staff","Env":["A=one"],"WorkingDir":"/workspace/project","Volumes":{"/workspace/project/cache":{}}}}"#, )], ); let client = PodmanClient::new(socket_path.clone()); @@ -1242,6 +1250,20 @@ mod tests { image.config.as_ref().map(|config| config.user.as_str()), Some("app:staff") ); + assert_eq!( + image + .config + .as_ref() + .map(|config| config.working_dir.as_str()), + Some("/workspace/project") + ); + assert!( + image + .config + .as_ref() + .and_then(|config| config.volumes.as_ref()) + .is_some_and(|volumes| volumes.contains_key("/workspace/project/cache")) + ); handle.await.expect("stub task should finish"); assert_eq!( request_log diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index dfd8b29443..f30fa717a8 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -3,6 +3,7 @@ //! Container spec construction for the Podman driver. +use crate::client::ImageInspect; use crate::config::PodmanComputeConfig; use openshell_core::ComputeDriverError; use openshell_core::driver_mounts::SelinuxLabel; @@ -49,6 +50,10 @@ const CONTAINER_PREFIX: &str = "openshell-"; /// Volume name prefix. const VOLUME_PREFIX: &str = "openshell-sandbox-"; +const PODMAN_WORKLOAD_CONTROL_PATHS: &[&str] = &[ + "/.openshell", + openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, +]; /// Secret name prefix for per-sandbox gateway JWTs. const TOKEN_SECRET_PREFIX: &str = "openshell-token-"; @@ -209,6 +214,74 @@ pub fn short_id(id: &str) -> String { id.chars().take(12).collect() } +/// Immutable OCI image metadata normalized once for final launch. +#[derive(Debug, Clone)] +pub struct ResolvedPodmanImage { + pub(crate) id: String, + pub(crate) oci_user: String, + pub(crate) environment: Vec, + pub(crate) workspace_root: String, +} + +impl ResolvedPodmanImage { + /// Resolve the image metadata and reject: + /// - a malformed or reserved working directory; + /// - for a custom working directory, an image volume with an invalid or + /// reserved target, or one covering the resolved workspace. + pub fn from_inspect(inspected: &ImageInspect) -> Result { + let image_config = inspected.config.as_ref(); + let workspace_root = driver_mounts::resolve_oci_workspace_root( + image_config.map_or("", |config| config.working_dir.as_str()), + ) + .map_err(ComputeDriverError::Precondition)?; + for control_path in PODMAN_WORKLOAD_CONTROL_PATHS { + driver_mounts::validate_workspace_control_path(&workspace_root, control_path) + .map_err(ComputeDriverError::Precondition)?; + } + if workspace_root != driver_mounts::DEFAULT_WORKSPACE_ROOT + && let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) + { + for volume in volumes.keys() { + validate_podman_mount_target(volume).map_err(|error| { + ComputeDriverError::Precondition(format!( + "invalid image-declared volume '{volume}': {error}" + )) + })?; + driver_mounts::validate_workspace_mount_target(volume, &workspace_root).map_err( + |_| { + ComputeDriverError::Precondition(format!( + "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" + )) + }, + )?; + } + } + Ok(Self { + id: inspected.id.clone(), + oci_user: image_config + .map_or("", |config| config.user.as_str()) + .to_string(), + environment: image_config.map_or_else(Vec::new, |config| config.env.clone()), + workspace_root, + }) + } + + /// Image reference without inspected metadata; uses the managed workspace. + #[cfg(test)] + fn unpinned(image: &str) -> Self { + Self { + id: image.to_string(), + oci_user: String::new(), + environment: Vec::new(), + workspace_root: driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), + } + } + + pub(crate) fn uses_managed_workspace(&self) -> bool { + self.workspace_root == driver_mounts::DEFAULT_WORKSPACE_ROOT + } +} + // --------------------------------------------------------------------------- // Typed container spec structs for the Podman libpod create API. // --------------------------------------------------------------------------- @@ -222,6 +295,11 @@ pub struct ContainerSpec { volumes: Vec, image_volumes: Vec, hostname: String, + /// Start trusted runtime binaries from `/` instead of the image + /// `WORKDIR`, so Podman neither creates a missing workdir nor fails to + /// `chdir` before `OpenShell` validates it. Children use the resolved + /// workspace. + work_dir: String, /// Overrides the image's ENTRYPOINT. In Podman's libpod API, `command` /// only overrides CMD (appended as args to the entrypoint). We must set /// `entrypoint` explicitly so the supervisor binary runs directly, @@ -875,6 +953,11 @@ fn podman_driver_config( Ok(config) } +fn validate_podman_mount_target(target: &str) -> Result<(), String> { + driver_mounts::validate_container_mount_target(target)?; + driver_mounts::validate_mount_control_path(target, "/.openshell") +} + fn validate_podman_driver_mounts( mounts: &[PodmanDriverMountConfig], enable_bind_mounts: bool, @@ -924,8 +1007,7 @@ fn validate_podman_driver_mounts( target } }; - driver_mounts::validate_container_mount_target(target)?; - driver_mounts::validate_mount_control_path(target, "/.openshell")?; + validate_podman_mount_target(target)?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(format!( @@ -1049,14 +1131,14 @@ pub fn build_container_spec_with_token_and_gpu_devices( gpu_device_ids: Option<&[String]>, ) -> Result { let image = resolve_image(sandbox, config); + let resolved_image = ResolvedPodmanImage::unpinned(image); build_container_spec_for_image( sandbox, config, token_secret_name, gpu_device_ids, image, - image, - "", + &resolved_image, None, None, ) @@ -1070,8 +1152,7 @@ pub fn build_container_spec_for_image( token_secret_name: Option<&str>, gpu_device_ids: Option<&[String]>, requested_image: &str, - image_id: &str, - oci_user: &str, + image: &ResolvedPodmanImage, supervisor_bin_path: Option<&Path>, tls_secret_names: Option<&[String; 1]>, ) -> Result { @@ -1081,8 +1162,7 @@ pub fn build_container_spec_for_image( token_secret_name, gpu_device_ids, requested_image, - image_id, - oci_user, + image, supervisor_bin_path, tls_secret_names, )?) @@ -1096,15 +1176,14 @@ fn build_base_spec( token_secret_name: Option<&str>, gpu_device_ids: Option<&[String]>, requested_image: &str, - image_id: &str, - oci_user: &str, + image: &ResolvedPodmanImage, supervisor_bin_path: Option<&Path>, tls_secret_names: Option<&[String; 1]>, ) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); - let env = build_env(sandbox, config, requested_image, oci_user)?; + let env = build_env(sandbox, config, requested_image, &image.oci_user)?; let mut labels = build_labels(sandbox); labels.insert( "openshell.ai/runtime-binary-source".into(), @@ -1115,6 +1194,13 @@ fn build_base_spec( let resource_limits = build_resource_limits(sandbox, config); let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) .map_err(ComputeDriverError::InvalidArgument)?; + let mount_targets = user_mounts.mounts.iter().map(|m| &m.destination); + let volume_targets = user_mounts.volumes.iter().map(|v| &v.dest); + let image_targets = user_mounts.image_volumes.iter().map(|v| &v.destination); + for target in mount_targets.chain(volume_targets).chain(image_targets) { + driver_mounts::validate_workspace_mount_target(target, &image.workspace_root) + .map_err(ComputeDriverError::Precondition)?; + } if sandbox .spec .as_ref() @@ -1138,11 +1224,14 @@ fn build_base_spec( let mut networks = BTreeMap::new(); networks.insert(config.network_name.clone(), NetworkAttachment {}); - let mut volumes = vec![NamedVolume { - name: vol, - dest: "/sandbox".into(), - options: vec!["rw".into()], - }]; + let mut volumes = Vec::new(); + if image.uses_managed_workspace() { + volumes.push(NamedVolume { + name: vol, + dest: driver_mounts::DEFAULT_WORKSPACE_ROOT.into(), + options: vec!["rw".into()], + }); + } volumes.extend(user_mounts.volumes); let mut image_volumes = if supervisor_bin_path.is_some() { @@ -1155,15 +1244,12 @@ fn build_base_spec( }] }; image_volumes.extend(user_mounts.image_volumes); - let mut command = vec![ - "--workdir".to_string(), - driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), - ]; + let mut command = vec!["--workdir".to_string(), image.workspace_root.clone()]; command.extend(upstream_proxy_cli_args(config)); let container_spec = ContainerSpec { name, - image: image_id.to_string(), + image: image.id.clone(), labels, env, volumes, @@ -1174,6 +1260,7 @@ fn build_base_spec( // /openshell-sandbox, so it appears at /opt/openshell/bin/openshell-sandbox. image_volumes, hostname: format!("sandbox-{}", sandbox.name), + work_dir: "/".into(), // Override the image's ENTRYPOINT so the supervisor binary runs // directly. Workload images can set // ENTRYPOINT ["/bin/bash"], and Podman's `command` field only @@ -1181,10 +1268,9 @@ fn build_base_spec( // Without this, the container would run the entrypoint binary with // the supervisor path as an argument instead of executing it directly. entrypoint: vec![SUPERVISOR_BINARY_PATH.into()], - // Keep Podman's existing /sandbox workspace contract explicit while - // the supervisor supports driver-selected workdirs. Operator-owned - // corporate proxy flags follow it; the workload command comes from - // the reserved environment variable. + // Pass the resolved workspace to the logical supervisor. Operator-owned + // corporate proxy flags follow it; the workload command comes from the + // reserved environment variable. command, // The paired builder supplies the immutable non-root identity. user: String::new(), @@ -1400,9 +1486,7 @@ pub struct IsolationSpecInput<'a> { pub resolver_secret: &'a str, pub gpu_devices: Option<&'a [String]>, pub requested_image: &'a str, - pub image_id: &'a str, - pub image_user: &'a str, - pub image_env: &'a [String], + pub image: &'a ResolvedPodmanImage, pub supervisor_bin: Option<&'a Path>, pub tls_secrets: Option<&'a [String; 1]>, pub identity: &'a openshell_isolation_interface::contract::ResolvedWorkloadIdentity, @@ -1437,8 +1521,7 @@ pub fn build_isolation_specs( input.token_secret, input.gpu_devices, input.requested_image, - input.image_id, - input.image_user, + input.image, input.supervisor_bin, input.tls_secrets, ) @@ -1453,7 +1536,8 @@ pub fn build_isolation_specs( .insert(crate::isolation::LABEL_ROLE.into(), "sandbox".into()); workload.env = BTreeMap::new(); workload.unsetenv = input - .image_env + .image + .environment .iter() .filter_map(|entry| entry.split_once('=').map(|(key, _)| key.to_string())) .collect(); @@ -1721,11 +1805,28 @@ fn parse_memory_to_bytes(quantity: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use crate::client::ImageConfig; use openshell_core::proto::compute::v1::{GpuResourceRequirements, ResourceRequirements}; static ENV_LOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + fn resolved_image(id: &str, user: &str, working_dir: &str) -> ResolvedPodmanImage { + ResolvedPodmanImage::from_inspect(&ImageInspect { + id: id.to_string(), + config: Some(ImageConfig { + user: user.to_string(), + env: vec![ + "LD_PRELOAD=/hostile.so".into(), + "HTTP_PROXY=http://bypass".into(), + ], + working_dir: working_dir.to_string(), + volumes: None, + }), + }) + .unwrap() + } + #[test] fn isolated_pair_keeps_privileges_network_and_secrets_out_of_workload() { let sandbox = DriverSandbox { @@ -1746,10 +1847,7 @@ mod tests { "sha256:image".into(), ) .unwrap(); - let env = vec![ - "LD_PRELOAD=/hostile.so".into(), - "HTTP_PROXY=http://bypass".into(), - ]; + let image = resolved_image("sha256:image", "1000:1001", ""); let specs = build_isolation_specs(IsolationSpecInput { sandbox: &sandbox, config: &config, @@ -1757,9 +1855,7 @@ mod tests { resolver_secret: "resolver", gpu_devices: None, requested_image: "image:latest", - image_id: "sha256:image", - image_user: "1000:1001", - image_env: &env, + image: &image, supervisor_bin: None, tls_secrets: None, identity: &identity, @@ -1813,9 +1909,7 @@ mod tests { resolver_secret: "resolver", gpu_devices: None, requested_image: "image:latest", - image_id: "sha256:image", - image_user: "", - image_env: &env, + image: &resolved_image("sha256:image", "", ""), supervisor_bin: None, tls_secrets: None, identity: &default_identity, @@ -1905,6 +1999,31 @@ mod tests { ); } + #[test] + fn resolved_image_rejects_volumes_covering_working_dir() { + let inspect = |volume: &str| ImageInspect { + id: "sha256:image".into(), + config: Some(ImageConfig { + working_dir: "/workspace/project".into(), + volumes: Some(std::collections::HashMap::from([( + volume.into(), + Value::Null, + )])), + ..Default::default() + }), + }; + + assert!(ResolvedPodmanImage::from_inspect(&inspect("/workspace")).is_err()); + let image = ResolvedPodmanImage::from_inspect(&inspect("/workspace/project/cache")) + .expect("image volumes nested below the workspace remain valid"); + assert_eq!(image.workspace_root, "/workspace/project"); + + let mut fallback = inspect("/etc/openshell"); + fallback.config.as_mut().unwrap().working_dir = "/sandbox".into(); + ResolvedPodmanImage::from_inspect(&fallback) + .expect("existing /sandbox images keep their image-volume behavior"); + } + fn json_struct(value: Value) -> prost_types::Struct { let Value::Object(object) = value else { panic!("expected JSON object"); @@ -2014,14 +2133,14 @@ mod tests { spec.environment.insert(key.to_string(), value.to_string()); } + let image = resolved_image("sha256:immutable", "app:staff", "/workspace/project"); let container = build_container_spec_for_image( &sandbox, &test_config(), None, None, "registry.example/app:latest", - "sha256:immutable", - "app:staff", + &image, None, None, ) @@ -2037,6 +2156,12 @@ mod tests { assert_eq!(container["image_pull_policy"].as_str(), Some("never")); assert_eq!(container["dns_search"], serde_json::json!([])); assert_eq!(container["dns_option"], serde_json::json!([])); + assert_eq!(container["work_dir"].as_str(), Some("/")); + assert_eq!( + container["command"], + serde_json::json!(["--workdir", "/workspace/project"]) + ); + assert!(container["volumes"].as_array().is_some_and(Vec::is_empty)); assert_eq!( container["env"][openshell_core::sandbox_env::OCI_IMAGE_USER].as_str(), Some("app:staff") @@ -2049,10 +2174,6 @@ mod tests { container["env"][openshell_core::sandbox_env::SANDBOX_GID].as_str(), Some("") ); - assert_eq!( - container["command"], - serde_json::json!(["--workdir", "/sandbox"]) - ); } #[test] @@ -2276,18 +2397,9 @@ mod tests { fn container_spec_defaults_drop_capabilities_and_keep_runtime_seccomp() { let sandbox = test_sandbox("test-id", "test-name"); let config = test_config(); - let spec = build_base_spec( - &sandbox, - &config, - None, - None, - "image", - "sha256:image", - "", - None, - None, - ) - .unwrap(); + let image = resolved_image("sha256:image", "", ""); + let spec = + build_base_spec(&sandbox, &config, None, None, "image", &image, None, None).unwrap(); assert_eq!(spec.cap_drop, vec!["ALL"]); assert!(spec.cap_add.is_empty()); assert!(spec.seccomp_profile_path.is_empty()); @@ -2997,6 +3109,39 @@ mod tests { ); } + #[test] + fn resolved_workspace_rejects_covering_driver_mount() { + use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; + + let image = resolved_image("sha256:immutable", "1000:1000", "/workspace/project"); + let mut sandbox = test_sandbox("test-id", "test-name"); + sandbox.spec = Some(DriverSandboxSpec { + template: Some(DriverSandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": "/workspace"}] + }))), + ..Default::default() + }), + ..Default::default() + }); + let error = build_container_spec_for_image( + &sandbox, + &test_config(), + None, + None, + "image:latest", + &image, + None, + None, + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("reserved for the OpenShell workspace") + ); + } + #[test] fn driver_config_rejects_bind_mounts_unless_enabled() { use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; @@ -3662,14 +3807,14 @@ mod tests { let sandbox = test_sandbox("bind-sv-id", "bind-sv-name"); let config = test_config(); let image = resolve_image(&sandbox, &config); + let resolved = resolved_image(image, "", ""); let spec = build_container_spec_for_image( &sandbox, &config, None, None, image, - image, - "", + &resolved, Some(Path::new("/host/cache/openshell-sandbox")), None, ) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index c7678a9c66..d99a3e7187 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -913,7 +913,7 @@ impl PodmanComputeDriver { "Creating sandbox container" ); - let (image, immutable_image_id, image_user, image_env) = async { + let (image, resolved_image) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { // The sandbox runtime is shipped in a standalone OCI image. @@ -971,10 +971,8 @@ impl PodmanComputeDriver { "podman image '{image}' inspection did not return an immutable image ID" ))); } - let image_user = inspected_image - .config - .as_ref() - .map_or_else(String::new, |config| config.user.clone()); + let resolved_image = + container::ResolvedPodmanImage::from_inspect(&inspected_image)?; for mount_image in container::podman_driver_image_mount_sources( sandbox, @@ -989,8 +987,7 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; } - let image_env = inspected_image.config.as_ref().map_or_else(Vec::new, |config| config.env.clone()); - Ok((image.to_string(), inspected_image.id, image_user, image_env)) + Ok((image.to_string(), resolved_image)) } .await; phase_status.finish(result) @@ -1001,6 +998,7 @@ impl PodmanComputeDriver { otel.status_code = tracing::field::Empty, )) .await?; + let managed_workspace = resolved_image.uses_managed_workspace(); // Fail closed on a missing/unreadable corporate proxy CA bundle before // creating any resources, so the operator gets a clear error @@ -1014,7 +1012,7 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; let identity = self - .resolve_workload_identity(sandbox, &immutable_image_id, &image_user) + .resolve_workload_identity(sandbox, &resolved_image.id, &resolved_image.oci_user) .await?; let channel_volume = crate::isolation::channel_volume_name(&sandbox.id); let mut runtime_config = self.config.clone(); @@ -1039,19 +1037,21 @@ impl PodmanComputeDriver { )); } - // Create the workspace volume and per-sandbox runtime files. + // Create the managed workspace volume, if needed, and runtime files. let (resolver_secret_name, token_secret_name, proxy_auth_secret_name) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { - self.client - .create_owned_volume( - &vol_name, - &sandbox.id, - &sandbox.workspace, - Some((identity.uid, identity.gid)), - ) - .await - .map_err(ComputeDriverError::from)?; + if managed_workspace { + self.client + .create_owned_volume( + &vol_name, + &sandbox.id, + &sandbox.workspace, + Some((identity.uid, identity.gid)), + ) + .await + .map_err(ComputeDriverError::from)?; + } let resolver_secret_name = match create_sandbox_resolver_secret(&self.client, &sandbox.id).await { Ok(name) => name, @@ -1171,9 +1171,7 @@ impl PodmanComputeDriver { resolver_secret: &resolver_secret_name, gpu_devices: gpu_devices.as_deref(), requested_image: &image, - image_id: &immutable_image_id, - image_user: &image_user, - image_env: &image_env, + image: &resolved_image, supervisor_bin: supervisor_bin_path.as_deref(), tls_secrets: tls_secret_names.as_ref(), identity: &identity, @@ -1198,7 +1196,7 @@ impl PodmanComputeDriver { created_workload = Some(workload_id.clone()); self.client.verify_isolation_fence(&workload_id).await?; self.admit_container_resources(&workload_id).await?; - let child_env = podman_child_environment(sandbox, &image_env); + let child_env = podman_child_environment(sandbox, &resolved_image.environment); let launch_authentication = sandbox .spec .as_ref() @@ -1234,9 +1232,11 @@ impl PodmanComputeDriver { archives.channel, ) .await?; - self.client - .copy_to_container(&workload_id, "/sandbox", archives.workspace) - .await?; + if managed_workspace { + self.client + .copy_to_container(&workload_id, "/sandbox", archives.workspace) + .await?; + } let supervisor_id = self .client .create_typed_container(&specs.supervisor) diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index d7364d4f6c..27c459c72c 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -29,6 +29,7 @@ const CONDITION_STARTING: &str = "ContainerStarting"; use openshell_core::driver_utils::{ CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_STOPPED, CONDITION_WORKSPACE_VALIDATION_FAILED, SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, + WORKSPACE_VALIDATION_FAILED_MESSAGE, }; pub type WatchStream = @@ -390,7 +391,12 @@ pub async fn inspect_workload( // Both containers exist before initial start. A missing or exited // companion therefore requires containment, including after a // gateway restart that missed the original Podman exit event. - Ok(_) | Err(PodmanApiError::NotFound(_)) => { + Ok(supervisor) => { + client.stop_container(&workload.id, 0).await?; + workload = client.inspect_container(&workload.id).await?; + workload.state.supervisor_exit_code = exited_exit_code(&supervisor.state); + } + Err(PodmanApiError::NotFound(_)) => { client.stop_container(&workload.id, 0).await?; workload = client.inspect_container(&workload.id).await?; } @@ -402,10 +408,17 @@ pub async fn inspect_workload( .await .ok() .and_then(|logs| boundary_startup_termination_marker(&logs)); + workload.state.supervisor_exit_code = supervisor + .ok() + .and_then(|supervisor| exited_exit_code(&supervisor.state)); } Ok(workload) } +fn exited_exit_code(state: &ContainerState) -> Option { + matches!(state.status.as_str(), "exited" | "stopped").then_some(state.exit_code) +} + /// Extract only fixed, OpenShell-owned startup diagnostics from container /// output. Workload and supervisor output may contain secrets, so it must not /// be propagated to driver conditions or tracing. @@ -553,10 +566,12 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition { "OOMKilled", "Container was killed by the OOM killer".to_string(), ) - } else if state.exit_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) { + } else if state.supervisor_exit_code + == Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)) + { ( CONDITION_WORKSPACE_VALIDATION_FAILED, - "OCI WorkingDir is not usable by the sandbox identity".to_string(), + WORKSPACE_VALIDATION_FAILED_MESSAGE.to_string(), ) } else if matches!(state.exit_code, 137 | 143) { ( @@ -692,6 +707,7 @@ mod tests { health: None, started_at: Some("2026-08-12T16:38:58Z".to_string()), finished_at: Some("2026-08-12T16:39:13Z".to_string()), + supervisor_exit_code: None, startup_diagnostic: None, }; @@ -741,6 +757,7 @@ mod tests { }), started_at: Some("2026-04-14T10:00:00Z".to_string()), finished_at: None, + supervisor_exit_code: None, startup_diagnostic: None, }; let cond = condition_from_state(&state); @@ -760,6 +777,7 @@ mod tests { health: None, started_at: Some("2026-04-14T10:00:00Z".to_string()), finished_at: None, + supervisor_exit_code: None, startup_diagnostic: None, }; let cond = condition_from_state(&state); @@ -782,6 +800,7 @@ mod tests { }), started_at: Some("2026-04-14T10:00:00Z".to_string()), finished_at: None, + supervisor_exit_code: None, startup_diagnostic: None, }; let condition = condition_from_state(&state); @@ -800,6 +819,7 @@ mod tests { health: None, started_at: None, finished_at: Some("2026-04-14T11:00:00Z".to_string()), + supervisor_exit_code: None, startup_diagnostic: None, }; let cond = condition_from_state(&state); @@ -818,6 +838,7 @@ mod tests { health: None, started_at: None, finished_at: Some("2026-04-14T12:00:00Z".to_string()), + supervisor_exit_code: None, startup_diagnostic: None, }; let cond = condition_from_state(&state); @@ -836,6 +857,7 @@ mod tests { health: None, started_at: None, finished_at: Some("2026-04-14T12:00:00Z".to_string()), + supervisor_exit_code: None, startup_diagnostic: boundary_startup_termination_marker( b"untrusted workload output\nsandbox boundary received SIGTERM before supervisor confirmation\n", ), @@ -860,21 +882,30 @@ mod tests { #[test] fn condition_workspace_validation_exit_is_reported_explicitly() { - let state = ContainerState { + // The supervisor exits with the reserved status and the watcher then + // stops the workload, so the workload itself reports SIGKILL. + let mut state = ContainerState { status: "exited".to_string(), running: false, - exit_code: i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED), + exit_code: 137, oom_killed: false, health: None, started_at: None, finished_at: Some("2026-04-14T12:00:00Z".to_string()), + supervisor_exit_code: Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)), startup_diagnostic: None, }; let cond = condition_from_state(&state); assert_eq!(cond.reason, CONDITION_WORKSPACE_VALIDATION_FAILED); - assert!(cond.message.contains("WorkingDir")); + assert_eq!(cond.message, WORKSPACE_VALIDATION_FAILED_MESSAGE); + + // A workload's own exit with the reserved status is not a supervisor + // workspace rejection. + state.exit_code = i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED); + state.supervisor_exit_code = Some(1); + assert_eq!(condition_from_state(&state).reason, CONDITION_EXITED); } #[test] @@ -892,6 +923,7 @@ mod tests { health: None, started_at: None, finished_at: Some("2026-04-14T12:30:00Z".to_string()), + supervisor_exit_code: None, startup_diagnostic: None, }; let cond = condition_from_state(&state); diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs index f594fa35f2..f3493769b8 100644 --- a/crates/openshell-sandbox/src/delegated.rs +++ b/crates/openshell-sandbox/src/delegated.rs @@ -54,7 +54,7 @@ pub async fn spawn_workload( crate::process::validate_oci_workspace_as_effective_identity(std::path::Path::new( workspace_root, )) - .wrap_err("image workspace validation failed")?; + .wrap_err(openshell_core::driver_utils::WORKSPACE_VALIDATION_ERROR_CONTEXT)?; } #[cfg(target_os = "linux")] diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 4bfb469891..b7be3c6868 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -11,7 +11,7 @@ use clap::{Parser, ValueEnum}; use miette::{IntoDiagnostic, Result}; use openshell_isolation_interface::contract::BackendDescriptor; use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; -use tracing::{info, warn}; +use tracing::{error, info, warn}; use tracing_subscriber::EnvFilter; use tracing_subscriber::filter::LevelFilter; use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _}; @@ -454,6 +454,7 @@ fn main() -> Result<()> { args.main_exit_marker, )) .await + .or_else(workspace_validation_exit) } SupervisorRole::NetworkProxy => { let listen = args.listen.unwrap_or_else(|| ([127, 0, 0, 1], 3128).into()); @@ -478,10 +479,51 @@ fn main() -> Result<()> { std::process::exit(exit_code); } +/// Exit with the reserved status when the sandbox rejects the image working +/// directory, so the compute driver can report the specific failure. +fn workspace_validation_exit(error: miette::Report) -> Result { + use openshell_core::driver_utils::{ + SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_ERROR_CONTEXT, + }; + // Display keeps the message on one line; Debug may wrap it. + if !error + .to_string() + .contains(WORKSPACE_VALIDATION_ERROR_CONTEXT) + { + return Err(error); + } + error!("Image workspace validation failed"); + eprintln!("{error:?}"); + Ok(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn workspace_validation_failure_exits_with_reserved_status() { + use miette::WrapErr as _; + use openshell_core::driver_utils::{ + SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_ERROR_CONTEXT, + }; + // Mirror how the sandbox reports the rejection across the boundary. + let sandbox_error = Err::<(), _>(miette::miette!( + "workspace path component '/workspace/project' is not writable by the sandbox \ + identity in the image: Permission denied (os error 13)" + )) + .wrap_err(WORKSPACE_VALIDATION_ERROR_CONTEXT) + .unwrap_err(); + let error = miette::miette!( + "process error: boundary process leaf: start process supervisor leaf: {sandbox_error:?}" + ); + assert_eq!( + workspace_validation_exit(error).expect("reserved exit status"), + SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED + ); + assert!(workspace_validation_exit(miette::miette!("connect failed")).is_err()); + } + #[test] fn isolation_backend_is_the_default_role() { let directory = tempfile::tempdir().expect("temporary runtime descriptor directory"); diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 505f2cc69b..1fb749988a 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -299,4 +299,4 @@ Set `process.run_as_user` and `process.run_as_group` in the sandbox policy to ch | Kubernetes | OpenShift SCC namespace annotations, otherwise `1000`. Override with `sandbox_uid` and `sandbox_gid`. | | MicroVM | The image's `sandbox` account, otherwise `1000`. Override with `sandbox_uid` and `sandbox_gid`. | -On Docker, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR`, `/`, or `/sandbox` use `/sandbox`. Any other `WORKDIR` must exist in the image and be writable by the sandbox user. Podman, Kubernetes, and MicroVM always use `/sandbox`. +On Docker and Podman, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR`, `/`, or `/sandbox` use `/sandbox`. Any other `WORKDIR` must exist in the image and be writable by the sandbox user. Kubernetes and MicroVM always use `/sandbox`. diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index c0deb2c750..0ebdc2359e 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -39,6 +39,7 @@ PODMAN_CI_TESTS=( podman_corporate_proxy podman_gateway_start podman_host_gateway + podman_oci_identity provider_token_exchange ) diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index d78e4c7454..4b437109c0 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -10,7 +10,8 @@ //! sandbox from its mutable tag, and verifies both the child identity and the //! image ID recorded on the real sandbox container. This exercises the Podman //! API inspect → protected metadata → create path rather than only its unit -//! serialization boundaries. +//! serialization boundaries. Workspace behavior shared with Docker is covered +//! by the `oci-image` feature suite in `tests/suites/features`. use std::process::Stdio; @@ -54,7 +55,16 @@ impl ImageGuard { let containerfile = context.path().join("Containerfile"); std::fs::write( &containerfile, - format!("FROM {BASE_IMAGE}\nUSER {OCI_UID}:{OCI_GID}\n"), + format!( + r"FROM {BASE_IMAGE} +USER 0:0 +RUN mkdir -p /home/app/project && \ + chown {OCI_UID}:{OCI_GID} /home/app /home/app/project && \ + chmod 0700 /home/app /home/app/project +WORKDIR /home/app/project +USER {OCI_UID}:{OCI_GID} +" + ), ) .map_err(|err| format!("write Containerfile: {err}"))?; @@ -260,7 +270,7 @@ async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, contai assert_eq!( workload_user, format!("{OCI_UID}:{OCI_GID}"), - "the workload must start directly as the final OCI identity" + "a custom OCI workspace must start directly as the final identity" ); let supervisor_user = run_engine( &image.engine, @@ -300,6 +310,8 @@ async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, contai .unwrap(); assert!(!mounts.contains("/etc/openshell/tls")); assert!(!mounts.contains("/.openshell/supervisor")); + assert!(!mounts.lines().any(|path| path == "/home/app/project")); + assert!(!mounts.lines().any(|path| path == "/sandbox")); let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/runtime-descriptor.json"]).await.expect("workload cannot read either control credential set"); assert!(posture.contains("0000000000000000")); } diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 46d5abc254..6610d6c068 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -286,7 +286,7 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default image workdir. -- Docker also rejects an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. +- Docker and Podman also reject an image `VOLUME` or driver mount that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. - A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`.