From e073caa915d8c3bd1e0015f523c5cb4717c845d4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 14:35:49 -0700 Subject: [PATCH 01/20] feat(podman): honor OCI image working directories Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 58 ++- crates/openshell-driver-podman/README.md | 32 ++ crates/openshell-driver-podman/src/client.rs | 47 ++- .../openshell-driver-podman/src/container.rs | 397 +++++++++++++++--- crates/openshell-driver-podman/src/driver.rs | 31 +- docs/how-it-works/sandboxes/runtimes.mdx | 16 +- e2e/rust/tests/custom_image.rs | 2 + e2e/rust/tests/driver_config_volume.rs | 14 +- e2e/rust/tests/podman_oci_identity.rs | 157 ++++++- skills/debug-openshell-cluster/SKILL.md | 11 +- skills/openshell-cli/SKILL.md | 8 + 11 files changed, 651 insertions(+), 122 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index ec6fff115a..e20667c474 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -9,10 +9,12 @@ Podman provisions a paired workload and supervisor container using its native libpod API. The workload uses `network=none`; the external supervisor alone joins the configured network. A per-sandbox named volume carries their mutually authenticated gRPC Unix socket, with supervisor credentials kept in its separate -filesystem. Both containers run as the resolved non-root identity with all -capabilities dropped. They share only a user namespace for volume ownership, -not PID, mount, or network namespaces. Podman owns paired lifecycle and health; -the common protocol owns process, identity, TCP, DNS, and forwarding semantics. +filesystem. The supervisor and final sandbox runtime run as the resolved non-root +identity with all capabilities dropped. Only the managed `/sandbox` fallback uses +a trusted root bootstrap to prepare its driver-owned workspace before dropping +irreversibly to that identity. The containers do not share PID, mount, or network +namespaces. Podman owns paired lifecycle and health; the common protocol owns +process, identity, TCP, DNS, and forwarding semantics. ## Driver Contract @@ -305,7 +307,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| | Docker | Local development with Docker available. | Capability-free workload container. | Uses `network_mode=none`; a separate capability-free supervisor container mediates egress and access over a private daemon-local Unix socket volume. | -| Podman | Existing rootless driver. | Container. | Not converted by this isolation stack. | +| Podman | Local development with Podman available. | Capability-free workload container. | Uses `network=none`; a separate capability-free supervisor container mediates egress and access over a private Unix socket volume. | | Kubernetes | Cluster deployment through Helm. | Capability-free sandbox Pod. | Always creates a namespace-wide empty-egress workload NetworkPolicy and a separate capability-free supervisor Pod over mutually authenticated TLS. It requires an enforcing CNI and trusted sandbox namespace; the Kubernetes API does not attest policy enforcement. | | VM | Experimental microVM isolation. | Per-sandbox libkrun or QEMU VM. | The NIC-less guest runs `openshell-sandbox` as PID 1; host `openshell-supervisor` owns gateway networking and reaches the guest over vsock. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | @@ -390,7 +392,7 @@ Drivers deliver the two binaries to separate trust domains: | Runtime | Delivery model | |---|---| | Docker | A digest-pinned daemon-local volume supplies `openshell-sandbox`; the companion image runs `openshell-supervisor`. | -| Podman | Existing driver behavior; not converted by this stack. | +| Podman | A pinned runtime image supplies `openshell-sandbox`; a separate pinned companion image runs `openshell-supervisor`. | | Kubernetes | A non-root init container stages `openshell-sandbox` into a memory volume; a directly managed Pod runs `openshell-supervisor`. | | VM | `openshell-sandbox` is embedded in the guest rootfs; a separately digest-checked native `openshell-supervisor` runs on the host. | | Extension | Defined by the out-of-tree driver. | @@ -408,19 +410,45 @@ and supplementary-group set before creating the immutable workload: - Docker pins the image ID, resolves policy selectors against the image's `/etc/passwd` and `/etc/group`, and validates its OCI working directory. +- Podman pins the image ID, resolves policy selectors against the image's + `/etc/passwd` and `/etc/group`, and validates its OCI working directory. - Kubernetes uses platform-resolved numeric values, including OpenShift namespace ranges. - VM uses the configured numeric guest identity. -UID/GID zero and `u32::MAX` are invalid. The sandbox and every child start with -the resolved identity and zero capability masks; neither process performs an -in-workload UID transition. Identity-changing policy updates require sandbox -recreation, while other policy updates remain live. - -Docker uses an absolute OCI working directory as the workspace. Empty, root, -and explicit `/sandbox` values select `/sandbox`; other paths must already -exist without symlink or reserved-mount collisions and must be usable by the -resolved identity. Kubernetes and VM use `/sandbox`. +UID/GID zero and `u32::MAX` are invalid. Before any untrusted instruction runs, +the sandbox runtime and every child use the resolved identity with zero +capability masks. The managed Podman `/sandbox` fallback may start its trusted +runtime bootstrap as container root with narrowly scoped identity and ownership +capabilities; it prepares the driver-owned workspace and drops irreversibly to +the resolved identity before reading bootstrap material or accepting control +traffic. No untrusted child performs an identity transition. Identity-changing +policy updates require sandbox recreation, while other policy updates remain +live. + +Docker and Podman resolve OCI `Config.User` and `Config.WorkingDir` from one +immutable image inspection. Empty, root (`/`), and explicit `/sandbox` working +directories select the managed `/sandbox` compatibility workspace. A custom +working directory must be a normalized absolute path outside kernel runtime, +OpenShell control, and private channel paths. A driver-config mount cannot cover +the workspace root or one of its parents; mounts nested beneath the root remain +valid. Image-declared volumes follow the same collision rules. Malformed paths +and collisions fail before untrusted execution. + +Podman mounts its persistent named workspace volume at the resolved custom root +without `nocopy` or ownership-changing options, retaining Podman's normal +first-use copy-up from the workload image. The trusted sandbox runtime starts at +`/` directly as the final non-root identity with no capabilities, validates the +effective copied-up path, and only then permits untrusted execution. Every path +component must be a real traversable directory and the root must be writable; +OpenShell preserves its ownership and mode. Only the `/sandbox` fallback uses +the root/chown bootstrap and managed-workspace archive. + +The separate supervisor receives the resolved path as the logical +`AgentSpec.workdir` but does not mount or traverse the workload workspace. +Podman copy-up results can vary with rootless or rootful operation, user +namespaces, backing filesystems, and SELinux; an unusable effective path fails +closed. Kubernetes and VM continue to use `/sandbox`. ### Executable Identity Binding diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 133326a0ec..c287c970c4 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -90,6 +90,38 @@ binary extraction path. `supervisor_image` supplies the dynamically linked glibc `/openshell-supervisor` binary outside the workload. Image and request environment belong to agent children, never the supervisor process. +## OCI working directory + +The same immutable workload-image inspection resolves OCI `WorkingDir`. An +empty value, `/`, or explicit `/sandbox` selects OpenShell's managed +`/sandbox` compatibility workspace. Any other value must be a normalized +absolute path that does not overlap kernel runtime mounts, OpenShell control +paths, or the private `/.openshell` channel. +Image-declared volumes and driver-config mounts may be nested below the +workspace, but cannot replace it or one of its parents. + +For a custom root, the driver mounts the persistent workspace volume at the +resolved path without `nocopy` or ownership-changing volume options. Podman +performs its normal first-use copy-up from the image. The driver does not upload +its managed-workspace ownership archive, chown the copied directory, or change +its mode. The capability-free sandbox runtime starts directly as the final +non-root identity and validates the effective copied-up path before it releases +untrusted code. Every path component must be a real, traversable directory and +the final directory must be writable. Direct and later exec/SSH children use +the path as both cwd and `HOME`; `filesystem.include_workdir` grants that same +root when enabled. + +Only the `/sandbox` fallback uses the root-to-non-root bootstrap and ownership +archive needed to create a driver-managed workspace. The separate supervisor +container receives the resolved path as logical `AgentSpec.workdir`; it neither +mounts nor traverses the workload workspace. + +Podman copy-up details can vary across rootless/rootful services, user namespace +modes, backing filesystems, and SELinux configuration. OpenShell preserves the +effective custom-workspace ownership and mode and fails closed when the final +identity cannot use it. Validate custom images with the deployment's actual +Podman configuration. + ## Lifecycle and readiness Create builds both stopped containers and stages the private archives before diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 80e24513a3..7a8ec8246a 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -185,6 +185,10 @@ pub struct ImageConfig { pub user: String, #[serde(default)] pub env: Vec, + #[serde(default)] + pub working_dir: String, + #[serde(default)] + pub volumes: Option>, } /// A container summary returned by the list API. @@ -1158,12 +1162,12 @@ mod tests { } #[tokio::test] - async fn inspect_image_reads_immutable_id_and_oci_user() { + async fn inspect_image_reads_immutable_id_and_oci_config() { let (socket_path, request_log, handle) = spawn_podman_stub( "inspect-image", vec![StubResponse::new( StatusCode::OK, - r#"{"Id":"sha256:immutable","Config":{"User":"app:staff"}}"#, + r#"{"Id":"sha256:immutable","Config":{"User":"app:staff","Env":["A=one"],"WorkingDir":"/workspace/project","Volumes":{"/workspace/project/cache":{}}}}"#, )], ); let client = PodmanClient::new(socket_path.clone()); @@ -1178,6 +1182,24 @@ mod tests { image.config.as_ref().map(|config| config.user.as_str()), Some("app:staff") ); + assert_eq!( + image.config.as_ref().map(|config| config.env.as_slice()), + Some(["A=one".to_string()].as_slice()) + ); + assert_eq!( + image + .config + .as_ref() + .map(|config| config.working_dir.as_str()), + Some("/workspace/project") + ); + assert!( + image + .config + .as_ref() + .and_then(|config| config.volumes.as_ref()) + .is_some_and(|volumes| volumes.contains_key("/workspace/project/cache")) + ); handle.await.expect("stub task should finish"); assert_eq!( request_log @@ -1189,6 +1211,27 @@ mod tests { let _ = std::fs::remove_file(socket_path); } + #[tokio::test] + async fn inspect_image_accepts_null_oci_volumes() { + let (socket_path, _request_log, handle) = spawn_podman_stub( + "inspect-image-null-volumes", + vec![StubResponse::new( + StatusCode::OK, + r#"{"Id":"sha256:immutable","Config":{"WorkingDir":"/workspace","Volumes":null}}"#, + )], + ); + let client = PodmanClient::new(socket_path.clone()); + + let image = client + .inspect_image("example/image:latest") + .await + .expect("null OCI volumes should parse"); + + assert!(image.config.is_some_and(|config| config.volumes.is_none())); + handle.await.expect("stub task should finish"); + let _ = std::fs::remove_file(socket_path); + } + #[tokio::test] async fn remove_container_uses_single_timed_libpod_removal() { let (socket_path, request_log, handle) = spawn_podman_stub( diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index bb4ee88c9f..f6372b36a8 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -3,6 +3,7 @@ //! Container spec construction for the Podman driver. +use crate::client::ImageInspect; use crate::config::PodmanComputeConfig; use openshell_core::ComputeDriverError; use openshell_core::driver_mounts::SelinuxLabel; @@ -217,6 +218,58 @@ pub fn short_id(id: &str) -> String { id.chars().take(12).collect() } +/// Immutable OCI image metadata normalized once for final launch. +#[derive(Debug, Clone)] +pub struct ResolvedPodmanImage { + pub(crate) id: String, + pub(crate) oci_user: String, + pub(crate) environment: Vec, + pub(crate) workspace_root: String, +} + +impl ResolvedPodmanImage { + pub fn from_inspect(inspected: &ImageInspect) -> Result { + let image_config = inspected.config.as_ref(); + let workspace_root = driver_mounts::resolve_oci_workspace_root( + image_config.map_or("", |config| config.working_dir.as_str()), + ) + .map_err(ComputeDriverError::Precondition)?; + driver_mounts::validate_workspace_control_path(&workspace_root, "/.openshell") + .map_err(ComputeDriverError::Precondition)?; + if let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) { + for volume in volumes.keys() { + driver_mounts::validate_container_mount_target(volume).map_err(|error| { + ComputeDriverError::Precondition(format!( + "invalid image-declared volume '{volume}': {error}" + )) + })?; + driver_mounts::validate_workspace_mount_target(volume, &workspace_root).map_err( + |_| { + ComputeDriverError::Precondition(format!( + "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" + )) + }, + )?; + driver_mounts::validate_mount_control_path(volume, "/.openshell") + .map_err(ComputeDriverError::Precondition)?; + } + } + + Ok(Self { + id: inspected.id.clone(), + oci_user: image_config + .map_or("", |config| config.user.as_str()) + .to_string(), + environment: image_config.map_or_else(Vec::new, |config| config.env.clone()), + workspace_root, + }) + } + + pub(crate) fn uses_managed_workspace(&self) -> bool { + self.workspace_root == driver_mounts::DEFAULT_WORKSPACE_ROOT + } +} + // --------------------------------------------------------------------------- // Typed container spec structs for the Podman libpod create API. // --------------------------------------------------------------------------- @@ -230,10 +283,11 @@ pub struct ContainerSpec { volumes: Vec, image_volumes: Vec, hostname: String, + /// Start trusted runtime binaries independently of the image-selected + /// workspace. The resolved workspace is applied to untrusted children. + work_dir: String, /// Overrides the image's ENTRYPOINT. In Podman's libpod API, `command` - /// only overrides CMD (appended as args to the entrypoint). We must set - /// `entrypoint` explicitly so the supervisor binary runs directly, - /// regardless of what ENTRYPOINT the sandbox image defines. + /// only overrides CMD (appended as args to the entrypoint). entrypoint: Vec, command: Vec, user: String, @@ -776,6 +830,7 @@ pub fn podman_driver_image_mount_sources( fn podman_user_mounts( sandbox: &DriverSandbox, enable_bind_mounts: bool, + workspace_root: &str, ) -> Result { let template = sandbox .spec @@ -787,6 +842,13 @@ fn podman_user_mounts( let config = podman_driver_config(template, enable_bind_mounts)?; let mut result = PodmanUserMounts::default(); for mount in config.mounts { + let target = match &mount { + PodmanDriverMountConfig::Bind { target, .. } + | PodmanDriverMountConfig::Volume { target, .. } + | PodmanDriverMountConfig::Tmpfs { target, .. } + | PodmanDriverMountConfig::Image { target, .. } => target, + }; + driver_mounts::validate_workspace_mount_target(target, workspace_root)?; match mount { PodmanDriverMountConfig::Bind { source, @@ -1062,14 +1124,17 @@ pub fn build_container_spec_with_token_and_gpu_devices( gpu_device_ids: Option<&[String]>, ) -> Result { let image = resolve_image(sandbox, config); + let resolved_image = ResolvedPodmanImage::from_inspect(&ImageInspect { + id: image.to_string(), + config: None, + })?; build_container_spec_for_image( sandbox, config, token_secret_name, gpu_device_ids, image, - image, - "", + &resolved_image, None, None, ) @@ -1083,8 +1148,7 @@ pub fn build_container_spec_for_image( token_secret_name: Option<&str>, gpu_device_ids: Option<&[String]>, requested_image: &str, - image_id: &str, - oci_user: &str, + image: &ResolvedPodmanImage, supervisor_bin_path: Option<&Path>, tls_secret_names: Option<&[String; 3]>, ) -> Result { @@ -1094,8 +1158,7 @@ pub fn build_container_spec_for_image( token_secret_name, gpu_device_ids, requested_image, - image_id, - oci_user, + image, supervisor_bin_path, tls_secret_names, )?) @@ -1109,15 +1172,14 @@ fn build_base_spec( token_secret_name: Option<&str>, gpu_device_ids: Option<&[String]>, requested_image: &str, - image_id: &str, - oci_user: &str, + image: &ResolvedPodmanImage, supervisor_bin_path: Option<&Path>, tls_secret_names: Option<&[String; 3]>, ) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); - let env = build_env(sandbox, config, requested_image, oci_user)?; + let env = build_env(sandbox, config, requested_image, &image.oci_user)?; let mut labels = build_labels(sandbox); labels.insert( "openshell.ai/runtime-binary-source".into(), @@ -1126,7 +1188,7 @@ fn build_base_spec( .unwrap_or_default(), ); let resource_limits = build_resource_limits(sandbox, config); - let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) + let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts, &image.workspace_root) .map_err(ComputeDriverError::InvalidArgument)?; if sandbox .spec @@ -1153,7 +1215,7 @@ fn build_base_spec( let mut volumes = vec![NamedVolume { name: vol, - dest: "/sandbox".into(), + dest: image.workspace_root.clone(), options: vec!["rw".into()], }]; volumes.extend(user_mounts.volumes); @@ -1168,15 +1230,12 @@ fn build_base_spec( }] }; image_volumes.extend(user_mounts.image_volumes); - let mut command = vec![ - "--workdir".to_string(), - driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), - ]; + let mut command = vec!["--workdir".to_string(), image.workspace_root.clone()]; command.extend(upstream_proxy_cli_args(config)); let container_spec = ContainerSpec { name, - image: image_id.to_string(), + image: image.id.clone(), labels, env, volumes, @@ -1187,6 +1246,7 @@ fn build_base_spec( // /openshell-sandbox, so it appears at /opt/openshell/bin/openshell-sandbox. image_volumes, hostname: format!("sandbox-{}", sandbox.name), + work_dir: "/".into(), // Override the image's ENTRYPOINT so the supervisor binary runs // directly. Workload images can set // ENTRYPOINT ["/bin/bash"], and Podman's `command` field only @@ -1194,10 +1254,9 @@ fn build_base_spec( // Without this, the container would run the entrypoint binary with // the supervisor path as an argument instead of executing it directly. entrypoint: vec![SUPERVISOR_BINARY_PATH.into()], - // Keep Podman's existing /sandbox workspace contract explicit while - // the supervisor supports driver-selected workdirs. Operator-owned - // corporate proxy flags follow it; the workload command comes from - // the reserved environment variable. + // Pass the resolved workspace to the logical supervisor. Operator-owned + // corporate proxy flags follow it; the workload command comes from the + // reserved environment variable. command, // The paired builder supplies the immutable non-root identity. user: String::new(), @@ -1439,9 +1498,7 @@ pub struct IsolationSpecInput<'a> { pub resolver_secret: &'a str, pub gpu_devices: Option<&'a [String]>, pub requested_image: &'a str, - pub image_id: &'a str, - pub image_user: &'a str, - pub image_env: &'a [String], + pub image: &'a ResolvedPodmanImage, pub supervisor_bin: Option<&'a Path>, pub tls_secrets: Option<&'a [String; 3]>, pub identity: &'a openshell_isolation_interface::contract::ResolvedWorkloadIdentity, @@ -1478,8 +1535,7 @@ pub fn build_isolation_specs( input.token_secret, input.gpu_devices, input.requested_image, - input.image_id, - input.image_user, + input.image, input.supervisor_bin, input.tls_secrets, ) @@ -1494,11 +1550,14 @@ pub fn build_isolation_specs( .insert(crate::isolation::LABEL_ROLE.into(), "sandbox".into()); workload.env = BTreeMap::new(); workload.unsetenv = input - .image_env + .image + .environment .iter() .filter_map(|entry| entry.split_once('=').map(|(key, _)| key.to_string())) .collect(); - if input.rootless || input.identity.source == "default" { + if input.image.uses_managed_workspace() + && (input.rootless || input.identity.source == "default") + { // Podman's archive endpoint leaves named-volume contents owned by // container root for rootless services and for a rootful USER-less // image's newly-created workspace. Start the trusted runtime as root @@ -1510,7 +1569,7 @@ pub fn build_isolation_specs( input.identity.uid.to_string(), input.identity.gid.to_string(), crate::isolation::BOOTSTRAP_PATH.into(), - driver_mounts::DEFAULT_WORKSPACE_ROOT.into(), + input.image.workspace_root.clone(), ]; workload.user = "0:0".into(); workload.groups.clear(); @@ -1791,11 +1850,28 @@ fn parse_memory_to_bytes(quantity: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use crate::client::ImageConfig; use openshell_core::proto::compute::v1::{GpuResourceRequirements, ResourceRequirements}; static ENV_LOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + fn resolved_image(id: &str, user: &str, working_dir: &str) -> ResolvedPodmanImage { + ResolvedPodmanImage::from_inspect(&ImageInspect { + id: id.to_string(), + config: Some(ImageConfig { + user: user.to_string(), + env: vec![ + "LD_PRELOAD=/hostile.so".into(), + "HTTP_PROXY=http://bypass".into(), + ], + working_dir: working_dir.to_string(), + volumes: None, + }), + }) + .unwrap() + } + #[test] fn isolated_pair_keeps_privileges_network_and_secrets_out_of_workload() { let sandbox = DriverSandbox { @@ -1816,10 +1892,7 @@ mod tests { "sha256:image".into(), ) .unwrap(); - let env = vec![ - "LD_PRELOAD=/hostile.so".into(), - "HTTP_PROXY=http://bypass".into(), - ]; + let image = resolved_image("sha256:image", "1000:1001", ""); let specs = build_isolation_specs(IsolationSpecInput { sandbox: &sandbox, config: &config, @@ -1827,9 +1900,7 @@ mod tests { resolver_secret: "resolver", gpu_devices: None, requested_image: "image:latest", - image_id: "sha256:image", - image_user: "1000:1001", - image_env: &env, + image: &image, supervisor_bin: None, tls_secrets: None, identity: &identity, @@ -1891,9 +1962,7 @@ mod tests { resolver_secret: "resolver", gpu_devices: None, requested_image: "image:latest", - image_id: "sha256:image", - image_user: "", - image_env: &env, + image: &resolved_image("sha256:image", "", ""), supervisor_bin: None, tls_secrets: None, identity: &default_identity, @@ -1989,6 +2058,141 @@ mod tests { ); } + #[test] + fn custom_workspace_stays_on_the_final_unprivileged_runtime_path() { + let sandbox = DriverSandbox { + id: "pair".into(), + name: "agent".into(), + ..Default::default() + }; + let config = PodmanComputeConfig::default(); + let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![2000], + "default".into(), + "sha256:image".into(), + ) + .unwrap(); + let image = resolved_image("sha256:image", "", "/workspace/project"); + + let specs = build_isolation_specs(IsolationSpecInput { + sandbox: &sandbox, + config: &config, + token_secret: None, + resolver_secret: "resolver", + gpu_devices: None, + requested_image: "image:latest", + image: &image, + supervisor_bin: None, + tls_secrets: None, + identity: &identity, + rootless: true, + }) + .unwrap(); + + assert_eq!(specs.workload.work_dir, "/"); + assert_eq!(specs.workload.user, "1000:1001"); + assert!(specs.workload.cap_add.is_empty()); + assert_eq!( + specs.workload.command, + vec!["--bootstrap", crate::isolation::BOOTSTRAP_PATH] + ); + assert!(specs.workload.volumes.iter().any(|volume| { + volume.name == volume_name("pair") && volume.dest == "/workspace/project" + })); + assert_eq!(specs.supervisor.work_dir, "/"); + assert_eq!( + &specs.supervisor.command[..2], + ["--workdir", "/workspace/project"] + ); + assert!( + specs + .supervisor + .volumes + .iter() + .all(|volume| volume.name != volume_name("pair")) + ); + } + + #[test] + fn resolved_image_applies_fallback_and_rejects_unsafe_workdirs() { + for working_dir in ["", "/", "/sandbox"] { + let image = resolved_image("sha256:image", "1000:1000", working_dir); + assert_eq!(image.workspace_root, "/sandbox"); + assert!(image.uses_managed_workspace()); + } + + for working_dir in [ + "relative/workspace", + "/workspace/../project", + "/proc/project", + "/.openshell", + "/.openshell/channel/project", + ] { + let inspected = ImageInspect { + id: "sha256:image".into(), + config: Some(ImageConfig { + user: "1000:1000".into(), + env: Vec::new(), + working_dir: working_dir.into(), + volumes: None, + }), + }; + assert!( + ResolvedPodmanImage::from_inspect(&inspected).is_err(), + "unsafe workdir {working_dir} should be rejected" + ); + } + + let image = resolved_image("sha256:image", "1000:1000", "/usr/src/app"); + assert_eq!(image.workspace_root, "/usr/src/app"); + assert!(!image.uses_managed_workspace()); + } + + #[test] + fn resolved_image_rejects_masking_oci_volumes_but_allows_nested_volumes() { + for volume in [ + "/workspace", + "/workspace/project", + "/.openshell", + "/.openshell/channel", + ] { + let inspected = ImageInspect { + id: "sha256:image".into(), + config: Some(ImageConfig { + user: "1000:1000".into(), + env: Vec::new(), + working_dir: "/workspace/project".into(), + volumes: Some(std::collections::HashMap::from([( + volume.to_string(), + Value::Object(serde_json::Map::default()), + )])), + }), + }; + assert!( + ResolvedPodmanImage::from_inspect(&inspected).is_err(), + "masking image volume {volume} should be rejected" + ); + } + + let inspected = ImageInspect { + id: "sha256:image".into(), + config: Some(ImageConfig { + user: "1000:1000".into(), + env: Vec::new(), + working_dir: "/workspace/project".into(), + volumes: Some(std::collections::HashMap::from([( + "/workspace/project/cache".into(), + Value::Object(serde_json::Map::default()), + )])), + }), + }; + let image = ResolvedPodmanImage::from_inspect(&inspected) + .expect("image volumes nested below the workspace remain valid"); + assert_eq!(image.workspace_root, "/workspace/project"); + } + fn json_struct(value: Value) -> prost_types::Struct { let Value::Object(object) = value else { panic!("expected JSON object"); @@ -2098,14 +2302,14 @@ mod tests { spec.environment.insert(key.to_string(), value.to_string()); } + let image = resolved_image("sha256:immutable", "app:staff", "/workspace/project"); let container = build_container_spec_for_image( &sandbox, &test_config(), None, None, "registry.example/app:latest", - "sha256:immutable", - "app:staff", + &image, None, None, ) @@ -2121,6 +2325,17 @@ mod tests { assert_eq!(container["image_pull_policy"].as_str(), Some("never")); assert_eq!(container["dns_search"], serde_json::json!([])); assert_eq!(container["dns_option"], serde_json::json!([])); + assert_eq!(container["work_dir"].as_str(), Some("/")); + assert_eq!( + container["command"], + serde_json::json!(["--workdir", "/workspace/project"]) + ); + assert!(container["volumes"].as_array().is_some_and(|volumes| { + volumes.iter().any(|volume| { + volume["name"].as_str() == Some("openshell-sandbox-test-id-workspace") + && volume["dest"].as_str() == Some("/workspace/project") + }) + })); assert_eq!( container["env"][openshell_core::sandbox_env::OCI_IMAGE_USER].as_str(), Some("app:staff") @@ -2133,10 +2348,6 @@ mod tests { container["env"][openshell_core::sandbox_env::SANDBOX_GID].as_str(), Some("") ); - assert_eq!( - container["command"], - serde_json::json!(["--workdir", "/sandbox"]) - ); } #[test] @@ -2360,18 +2571,9 @@ mod tests { fn container_spec_defaults_drop_capabilities_and_keep_runtime_seccomp() { let sandbox = test_sandbox("test-id", "test-name"); let config = test_config(); - let spec = build_base_spec( - &sandbox, - &config, - None, - None, - "image", - "sha256:image", - "", - None, - None, - ) - .unwrap(); + let image = resolved_image("sha256:image", "", ""); + let spec = + build_base_spec(&sandbox, &config, None, None, "image", &image, None, None).unwrap(); assert_eq!(spec.cap_drop, vec!["ALL"]); assert!(spec.cap_add.is_empty()); assert!(spec.seccomp_profile_path.is_empty()); @@ -3081,6 +3283,79 @@ mod tests { ); } + #[test] + fn resolved_workspace_rejects_covering_mounts_but_allows_nested_mounts() { + use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; + + let image = resolved_image("sha256:immutable", "1000:1000", "/workspace/project"); + for target in ["/workspace", "/workspace/project"] { + let mut sandbox = test_sandbox("test-id", "test-name"); + sandbox.spec = Some(DriverSandboxSpec { + template: Some(DriverSandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": target}] + }))), + ..Default::default() + }), + ..Default::default() + }); + + let error = build_container_spec_for_image( + &sandbox, + &test_config(), + None, + None, + "image:latest", + &image, + None, + None, + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("reserved for the OpenShell workspace"), + "covering target {target} should be rejected: {error}" + ); + } + + let mut sandbox = test_sandbox("test-id", "test-name"); + sandbox.spec = Some(DriverSandboxSpec { + template: Some(DriverSandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "mounts": [ + {"type": "tmpfs", "target": "/workspace/project/cache"}, + {"type": "tmpfs", "target": "/sandbox"} + ] + }))), + ..Default::default() + }), + ..Default::default() + }); + let spec = build_container_spec_for_image( + &sandbox, + &test_config(), + None, + None, + "image:latest", + &image, + None, + None, + ) + .expect("nested and unrelated mounts should remain valid"); + let mounts = spec["mounts"].as_array().unwrap(); + assert!( + mounts + .iter() + .any(|mount| mount["destination"] == "/workspace/project/cache") + ); + assert!( + mounts + .iter() + .any(|mount| mount["destination"] == "/sandbox") + ); + } + #[test] fn driver_config_rejects_bind_mounts_unless_enabled() { use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; @@ -3755,14 +4030,14 @@ mod tests { let sandbox = test_sandbox("bind-sv-id", "bind-sv-name"); let config = test_config(); let image = resolve_image(&sandbox, &config); + let resolved = resolved_image(image, "", ""); let spec = build_container_spec_for_image( &sandbox, &config, None, None, image, - image, - "", + &resolved, Some(Path::new("/host/cache/openshell-sandbox")), None, ) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 3788c756f4..4af454ec54 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -905,7 +905,7 @@ impl PodmanComputeDriver { "Creating sandbox container" ); - let (image, immutable_image_id, image_user, image_env) = async { + let (image, resolved_image) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { // The sandbox runtime is shipped in a standalone OCI image. @@ -963,10 +963,8 @@ impl PodmanComputeDriver { "podman image '{image}' inspection did not return an immutable image ID" ))); } - let image_user = inspected_image - .config - .as_ref() - .map_or_else(String::new, |config| config.user.clone()); + let resolved_image = + container::ResolvedPodmanImage::from_inspect(&inspected_image)?; for mount_image in container::podman_driver_image_mount_sources( sandbox, @@ -981,8 +979,7 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; } - let image_env = inspected_image.config.as_ref().map_or_else(Vec::new, |config| config.env.clone()); - Ok((image.to_string(), inspected_image.id, image_user, image_env)) + Ok((image.to_string(), resolved_image)) } .await; phase_status.finish(result) @@ -1006,7 +1003,7 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; let identity = self - .resolve_workload_identity(sandbox, &immutable_image_id, &image_user) + .resolve_workload_identity(sandbox, &resolved_image.id, &resolved_image.oci_user) .await?; let channel_volume = crate::isolation::channel_volume_name(&sandbox.id); let mut runtime_config = self.config.clone(); @@ -1158,9 +1155,7 @@ impl PodmanComputeDriver { resolver_secret: &resolver_secret_name, gpu_devices: gpu_devices.as_deref(), requested_image: &image, - image_id: &immutable_image_id, - image_user: &image_user, - image_env: &image_env, + image: &resolved_image, supervisor_bin: supervisor_bin_path.as_deref(), tls_secrets: tls_secret_names.as_ref(), identity: &identity, @@ -1186,7 +1181,7 @@ impl PodmanComputeDriver { created_workload = Some(workload_id.clone()); self.client.verify_isolation_fence(&workload_id).await?; self.admit_container_resources(&workload_id).await?; - let child_env = podman_child_environment(sandbox, &image_env); + let child_env = podman_child_environment(sandbox, &resolved_image.environment); let launch_authentication = sandbox .spec .as_ref() @@ -1222,9 +1217,15 @@ impl PodmanComputeDriver { archives.channel, ) .await?; - self.client - .copy_to_container(&workload_id, "/sandbox", archives.workspace) - .await?; + if resolved_image.uses_managed_workspace() { + self.client + .copy_to_container( + &workload_id, + &resolved_image.workspace_root, + archives.workspace, + ) + .await?; + } let supervisor_id = self .client .create_typed_container(&specs.supervisor) diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index cd3d576b52..0cf5d65e8b 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -267,4 +267,18 @@ Set `process.run_as_user` and `process.run_as_group` in the sandbox policy to ch | Kubernetes | OpenShift SCC namespace annotations, otherwise `1000`. Override with `sandbox_uid` and `sandbox_gid`. | | MicroVM | The image's `sandbox` account, otherwise `1000`. Override with `sandbox_uid` and `sandbox_gid`. | -On Docker, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR`, `/`, or `/sandbox` use `/sandbox`. Any other `WORKDIR` must exist in the image and be writable by the sandbox user. Podman, Kubernetes, and MicroVM always use `/sandbox`. +On Docker and Podman, the image's `WORKDIR` becomes the workspace. Images with +no `WORKDIR`, `WORKDIR /`, or `WORKDIR /sandbox` use OpenShell's managed +`/sandbox` compatibility workspace. Any other value must be a normalized +absolute path outside kernel and OpenShell control mounts. Driver-config mounts +and image-declared volumes cannot replace that root, but mounts below it remain +valid. + +Docker validates the directory in the image filesystem. Podman mounts the +persistent workspace volume at the resolved path and preserves Podman's normal +first-use copy-up. OpenShell does not chown, chmod, or otherwise repair a custom +Podman workspace. The final non-root identity must be able to traverse and +write the copied-up directory before the workload starts. Copy-up ownership can +vary with rootless or rootful operation, user namespaces, the backing +filesystem, and SELinux, so validate custom images in the target Podman +environment. Kubernetes and MicroVM continue to use `/sandbox`. diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs index 96fdd56612..e2412f8ddf 100644 --- a/e2e/rust/tests/custom_image.rs +++ b/e2e/rust/tests/custom_image.rs @@ -49,6 +49,7 @@ USER 2345:2346 CMD ["sleep", "infinity"] "#; +#[cfg(feature = "e2e-docker")] const UNWRITABLE_WORKDIR_DOCKERFILE_CONTENT: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ @@ -223,6 +224,7 @@ async fn sandbox_from_passwd_less_numeric_oci_user() { #[tokio::test] #[serial(custom_image)] +#[cfg(feature = "e2e-docker")] async fn sandbox_rejects_image_workdir_that_would_require_new_authority() { let tmpdir = tempfile::tempdir().expect("create tmpdir"); let dockerfile_path = tmpdir.path().join("Dockerfile"); diff --git a/e2e/rust/tests/driver_config_volume.rs b/e2e/rust/tests/driver_config_volume.rs index bcef6a429c..9d35bd00d6 100644 --- a/e2e/rust/tests/driver_config_volume.rs +++ b/e2e/rust/tests/driver_config_volume.rs @@ -17,7 +17,7 @@ use bollard::query_parameters::{ RemoveVolumeOptionsBuilder, StartContainerOptions, WaitContainerOptions, }; use futures_util::TryStreamExt; -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] use openshell_e2e::harness::container::ImageGuard; use openshell_e2e::harness::container::e2e_driver; use openshell_e2e::harness::sandbox::SandboxGuard; @@ -26,9 +26,9 @@ use serde_json::{Map, Value}; const TEST_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; const VOLUME_TARGET: &str = "/sandbox/e2e-volume"; const BIND_TARGET: &str = "/sandbox/e2e-bind"; -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] const OCI_VOLUME_TARGET: &str = "/workspace/project/e2e-volume"; -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] const OCI_USER_DOCKERFILE: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ @@ -119,12 +119,12 @@ async fn sandbox_mounts_existing_driver_config_volume() { } #[tokio::test] -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] async fn oci_workspace_preparation_skips_nested_volume_ownership() { let driver = e2e_driver().expect("OPENSHELL_E2E_DRIVER must be set by the e2e wrapper"); assert!( - driver == "docker", - "OCI workspace mount e2e requires docker, got {driver}" + matches!(driver.as_str(), "docker" | "podman"), + "OCI workspace mount e2e requires docker or podman, got {driver}" ); let volume = VolumeGuard::create(&driver) @@ -305,7 +305,7 @@ async fn verify_volume(volume: &VolumeGuard) -> Result<(), String> { Ok(()) } -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] async fn verify_volume_ownership(volume: &VolumeGuard) -> Result<(), String> { let output = run_volume_container( volume, diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index 3405d730c9..f76f430a27 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -3,14 +3,13 @@ #![cfg(feature = "e2e-podman")] -//! Podman-specific E2E coverage for OCI identity inspection and immutable-image -//! launch. +//! Podman-specific E2E coverage for OCI identity/workspace inspection, +//! workspace-volume copy-up, and immutable-image launch. //! //! The test builds an image through the selected Podman engine, creates a -//! sandbox from its mutable tag, and verifies both the child identity and the -//! image ID recorded on the real sandbox container. This exercises the Podman -//! API inspect → protected metadata → create path rather than only its unit -//! serialization boundaries. +//! sandbox from its mutable tag, and verifies the child identity, copied image +//! content, workspace placement, and image ID recorded on the real sandbox +//! container. use std::process::Stdio; @@ -54,7 +53,17 @@ impl ImageGuard { let containerfile = context.path().join("Containerfile"); std::fs::write( &containerfile, - format!("FROM {BASE_IMAGE}\nUSER {OCI_UID}:{OCI_GID}\n"), + format!( + r"FROM {BASE_IMAGE} +USER 0:0 +RUN mkdir -p /home/app/project && \ + chown {OCI_UID}:{OCI_GID} /home/app /home/app/project && \ + chmod 0700 /home/app /home/app/project +WORKDIR /home/app/project +RUN printf root-owned > root-owned.txt && chown {OCI_UID}:{OCI_GID} . +USER {OCI_UID}:{OCI_GID} +" + ), ) .map_err(|err| format!("write Containerfile: {err}"))?; @@ -89,6 +98,71 @@ impl ImageGuard { "Podman-built image has OCI user '{user}', expected {OCI_UID}:{OCI_GID}" )); } + let working_dir = run_engine( + &engine, + &[ + "image", + "inspect", + "--format", + "{{.Config.WorkingDir}}", + &tag, + ], + )?; + if working_dir != "/home/app/project" { + return Err(format!( + "Podman-built image has OCI workdir '{working_dir}', expected /home/app/project" + )); + } + + Ok(Self { engine, tag, id }) + } + + fn build_unwritable() -> Result { + let engine = ContainerEngine::from_env()?; + if engine.name() != "podman" { + return Err(format!( + "Podman OCI workspace E2E requires podman, got {}", + engine.name() + )); + } + + let context = tempfile::tempdir().map_err(|err| format!("create build context: {err}"))?; + let containerfile = context.path().join("Containerfile"); + std::fs::write( + &containerfile, + format!( + r"FROM {BASE_IMAGE} +USER 0:0 +RUN mkdir -p /root-owned/project && chmod 0700 /root-owned /root-owned/project +WORKDIR /root-owned/project +USER {OCI_UID}:{OCI_GID} +" + ), + ) + .map_err(|err| format!("write Containerfile: {err}"))?; + + let tag = format!( + "localhost/openshell-e2e-podman-unwritable-workdir:{}", + std::process::id() + ); + run_engine( + &engine, + &[ + "build", + "--pull=never", + "--file", + containerfile + .to_str() + .ok_or_else(|| "Containerfile path is not UTF-8".to_string())?, + "--tag", + &tag, + context + .path() + .to_str() + .ok_or_else(|| "build context path is not UTF-8".to_string())?, + ], + )?; + let id = run_engine(&engine, &["image", "inspect", "--format", "{{.Id}}", &tag])?; Ok(Self { engine, tag, id }) } @@ -175,7 +249,7 @@ fn normalized_image_id(image_id: &str) -> &str { } #[tokio::test] -async fn podman_uses_oci_identity_and_inspected_image_id() { +async fn podman_uses_oci_identity_workspace_copy_up_and_inspected_image_id() { if !is_e2e_driver("podman") { eprintln!("Skipping Podman OCI identity test: e2e driver is not podman"); return; @@ -189,17 +263,20 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { std::fs::write(policy.path(), OCI_FALLBACK_POLICY).expect("write OCI fallback policy"); let policy_path = policy.path().to_str().expect("policy path is UTF-8"); let mut sandbox = SandboxGuard::create_keep_with_args( - &[ - "--from", - &image.tag, - "--policy", - policy_path, - "--no-tty", - ], + &["--from", &image.tag, "--policy", policy_path, "--no-tty"], &[ "sh", "-c", - "set -eu; printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; echo podman-oci-identity-ready; sleep infinity", + "set -eu; \ + test \"$(pwd -P)\" = /home/app/project; \ + test \"$HOME\" = /home/app/project; \ + test \"$(cat root-owned.txt)\" = root-owned; \ + test \"$(stat -c %u:%g .)\" = 2345:2346; \ + test \"$(stat -c %a .)\" = 700; \ + test \"$(stat -c %u:%g root-owned.txt)\" = 0:0; \ + touch direct-workspace-write; \ + printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; \ + echo podman-oci-identity-ready; sleep infinity", ], READY_MARKER, ) @@ -216,7 +293,13 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { .exec(&[ "sh", "-c", - "test \"$(id -u):$(id -g)\" = 2345:2346; echo podman-ssh-identity-ok", + "set -eu; \ + test \"$(id -u):$(id -g)\" = 2345:2346; \ + test \"$(pwd -P)\" = /home/app/project; \ + test \"$HOME\" = /home/app/project; \ + test -f direct-workspace-write; \ + touch ssh-workspace-write; \ + echo podman-ssh-identity-ok", ]) .await .expect("SSH child should use Podman OCI identity"); @@ -248,6 +331,39 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { sandbox.cleanup().await; } +#[tokio::test] +async fn podman_rejects_copied_workspace_unusable_by_final_identity() { + if !is_e2e_driver("podman") { + eprintln!("Skipping Podman OCI workspace rejection test: e2e driver is not podman"); + return; + } + + let image = ImageGuard::build_unwritable().expect("build unwritable Podman OCI image"); + let policy = tempfile::NamedTempFile::new().expect("create OCI fallback policy"); + std::fs::write(policy.path(), OCI_FALLBACK_POLICY).expect("write OCI fallback policy"); + let policy_path = policy.path().to_str().expect("policy path is UTF-8"); + let result = SandboxGuard::create_keep_with_args( + &["--from", &image.tag, "--policy", policy_path, "--no-tty"], + &["sh", "-c", "echo should-not-run"], + "should-not-run", + ) + .await; + let error = match result { + Ok(mut sandbox) => { + sandbox.cleanup().await; + panic!("a copied workspace unusable by the final identity must fail closed"); + } + Err(error) => error, + }; + let message = error; + assert!( + (message.contains("WorkspaceValidationFailed") && message.contains("WorkingDir")) + || message.contains("subsystem request failed") + || message.contains("image workspace validation failed"), + "expected copied workspace validation failure, got: {message}" + ); +} + async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, container_id: &str) { let supervisor_id = container_id_for_role(&image.engine, &sandbox.name, "supervisor") .expect("find separate supervisor companion"); @@ -258,8 +374,9 @@ async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, contai ) .unwrap(); assert_eq!( - workload_user, "0:0", - "the trusted rootless boundary starts as container root before dropping to the OCI identity" + workload_user, + format!("{OCI_UID}:{OCI_GID}"), + "a custom OCI workspace must start directly as the final identity" ); let supervisor_user = run_engine( &image.engine, @@ -299,6 +416,8 @@ async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, contai .unwrap(); assert!(!mounts.contains("/etc/openshell/tls")); assert!(!mounts.contains("/.openshell/supervisor")); + assert!(mounts.lines().any(|path| path == "/home/app/project")); + assert!(!mounts.lines().any(|path| path == "/sandbox")); let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/runtime-descriptor.json"]).await.expect("workload cannot read either control credential set"); assert!(posture.contains("0000000000000000")); } diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 85360a77ad..15f6af2225 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -279,8 +279,8 @@ Common findings: - Gateway process stopped: inspect exit status and logs. - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. -- Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default image workdir. -- Docker also rejects an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. +- Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default Docker workdir or a copied-up custom Podman workspace. +- Docker and Podman also reject an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the path before validation. Move the `VOLUME` below the workspace or remove the declaration. - A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. @@ -326,6 +326,13 @@ Common findings: - Rootless networking unavailable: inspect Podman network configuration. - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. +- A custom OCI `WORKDIR` uses the Podman workspace volume at that path. Inspect + the workload container's mounts and the copied directory ownership/mode. The + custom path starts directly as the final non-root identity; only managed + `/sandbox` uses OpenShell's root/chown bootstrap. Rootless/rootful services, + user namespaces, backing filesystems, and SELinux can produce different + copy-up results. Fix the image or runtime configuration instead of adding + capabilities or asking OpenShell to repair the custom path. - Supervisor cannot connect: check its gateway endpoint and gateway logs. - Inspect both Podman containers for the sandbox: the `sandbox` isolation role must have network mode `none`; the `supervisor` role owns the gateway session diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index 818d12a269..40149a3b0a 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -678,6 +678,14 @@ Explicit numeric fields may use any UID/GID from `1` through Warn users that low IDs can inherit permissions from matching accounts, image files, mounted volumes, or devices. +Docker and Podman gateways also use a normalized absolute OCI `WORKDIR` as the +workspace. Empty, `/`, and explicit `/sandbox` declarations use the managed +`/sandbox` fallback. Podman mounts its persistent workspace volume at a custom +workdir and preserves normal first-use image copy-up; it does not repair the +copied ownership or mode. Make the final non-root identity able to traverse and +write that directory, and test the image with the target rootless/rootful, +user-namespace, filesystem, and SELinux configuration. + ### Forward ports ```bash From 6e30504781605b8309f35e48d199209ec5b9adce Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 15:30:06 -0700 Subject: [PATCH 02/20] refactor(podman): trim workdir change set Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 32 +-- crates/openshell-driver-podman/README.md | 41 ++-- crates/openshell-driver-podman/src/client.rs | 25 --- .../openshell-driver-podman/src/container.rs | 204 +++--------------- docs/how-it-works/sandboxes/runtimes.mdx | 17 +- e2e/rust/Cargo.toml | 2 +- e2e/rust/tests/custom_image.rs | 4 +- e2e/rust/tests/podman_oci_identity.rs | 103 --------- skills/debug-openshell-cluster/SKILL.md | 7 - skills/openshell-cli/SKILL.md | 8 +- 10 files changed, 71 insertions(+), 372 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e20667c474..1cbc6088d0 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -427,28 +427,16 @@ policy updates require sandbox recreation, while other policy updates remain live. Docker and Podman resolve OCI `Config.User` and `Config.WorkingDir` from one -immutable image inspection. Empty, root (`/`), and explicit `/sandbox` working -directories select the managed `/sandbox` compatibility workspace. A custom -working directory must be a normalized absolute path outside kernel runtime, -OpenShell control, and private channel paths. A driver-config mount cannot cover -the workspace root or one of its parents; mounts nested beneath the root remain -valid. Image-declared volumes follow the same collision rules. Malformed paths -and collisions fail before untrusted execution. - -Podman mounts its persistent named workspace volume at the resolved custom root -without `nocopy` or ownership-changing options, retaining Podman's normal -first-use copy-up from the workload image. The trusted sandbox runtime starts at -`/` directly as the final non-root identity with no capabilities, validates the -effective copied-up path, and only then permits untrusted execution. Every path -component must be a real traversable directory and the root must be writable; -OpenShell preserves its ownership and mode. Only the `/sandbox` fallback uses -the root/chown bootstrap and managed-workspace archive. - -The separate supervisor receives the resolved path as the logical -`AgentSpec.workdir` but does not mount or traverse the workload workspace. -Podman copy-up results can vary with rootless or rootful operation, user -namespaces, backing filesystems, and SELinux; an unusable effective path fails -closed. Kubernetes and VM continue to use `/sandbox`. +immutable image inspection. Empty, `/`, and explicit `/sandbox` values use the +managed `/sandbox` workspace. Custom paths must be normalized absolute paths +outside runtime and control mounts; image and driver mounts cannot cover them. + +Podman mounts its persistent workspace volume at a custom root with normal +image copy-up, preserves the resulting ownership and mode, and starts directly +as the final non-root identity. An unusable path fails closed. Only the managed +`/sandbox` fallback uses the root/chown bootstrap. The separate supervisor uses +the path logically but does not mount the workspace. Kubernetes and VM continue +to use `/sandbox`. ### Executable Identity Binding diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index c287c970c4..63634ed1a9 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -92,35 +92,24 @@ environment belong to agent children, never the supervisor process. ## OCI working directory -The same immutable workload-image inspection resolves OCI `WorkingDir`. An -empty value, `/`, or explicit `/sandbox` selects OpenShell's managed -`/sandbox` compatibility workspace. Any other value must be a normalized -absolute path that does not overlap kernel runtime mounts, OpenShell control -paths, or the private `/.openshell` channel. -Image-declared volumes and driver-config mounts may be nested below the -workspace, but cannot replace it or one of its parents. +The immutable workload-image inspection also resolves OCI `WorkingDir`. Empty, +`/`, and explicit `/sandbox` values select the managed `/sandbox` workspace. +Custom paths must be normalized absolute paths outside runtime and control +mounts. Image and driver mounts may be nested below the workspace but cannot +cover it. For a custom root, the driver mounts the persistent workspace volume at the resolved path without `nocopy` or ownership-changing volume options. Podman -performs its normal first-use copy-up from the image. The driver does not upload -its managed-workspace ownership archive, chown the copied directory, or change -its mode. The capability-free sandbox runtime starts directly as the final -non-root identity and validates the effective copied-up path before it releases -untrusted code. Every path component must be a real, traversable directory and -the final directory must be writable. Direct and later exec/SSH children use -the path as both cwd and `HOME`; `filesystem.include_workdir` grants that same -root when enabled. - -Only the `/sandbox` fallback uses the root-to-non-root bootstrap and ownership -archive needed to create a driver-managed workspace. The separate supervisor -container receives the resolved path as logical `AgentSpec.workdir`; it neither -mounts nor traverses the workload workspace. - -Podman copy-up details can vary across rootless/rootful services, user namespace -modes, backing filesystems, and SELinux configuration. OpenShell preserves the -effective custom-workspace ownership and mode and fails closed when the final -identity cannot use it. Validate custom images with the deployment's actual -Podman configuration. +performs its normal first-use copy-up from the image. OpenShell preserves the +copied ownership and mode, starts directly as the final non-root identity, and +fails closed unless that identity can traverse and write the path. Agent +children use the path as cwd and `HOME`; `filesystem.include_workdir` grants it +when enabled. + +Only `/sandbox` uses the root-to-non-root bootstrap needed to prepare a managed +workspace. The separate supervisor receives the path logically but does not +mount it. Because Podman copy-up varies by deployment, validate custom images +with the target runtime configuration. ## Lifecycle and readiness diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 7a8ec8246a..84301f85d1 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -1182,10 +1182,6 @@ mod tests { image.config.as_ref().map(|config| config.user.as_str()), Some("app:staff") ); - assert_eq!( - image.config.as_ref().map(|config| config.env.as_slice()), - Some(["A=one".to_string()].as_slice()) - ); assert_eq!( image .config @@ -1211,27 +1207,6 @@ mod tests { let _ = std::fs::remove_file(socket_path); } - #[tokio::test] - async fn inspect_image_accepts_null_oci_volumes() { - let (socket_path, _request_log, handle) = spawn_podman_stub( - "inspect-image-null-volumes", - vec![StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:immutable","Config":{"WorkingDir":"/workspace","Volumes":null}}"#, - )], - ); - let client = PodmanClient::new(socket_path.clone()); - - let image = client - .inspect_image("example/image:latest") - .await - .expect("null OCI volumes should parse"); - - assert!(image.config.is_some_and(|config| config.volumes.is_none())); - handle.await.expect("stub task should finish"); - let _ = std::fs::remove_file(socket_path); - } - #[tokio::test] async fn remove_container_uses_single_timed_libpod_removal() { let (socket_path, request_log, handle) = spawn_podman_stub( diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index f6372b36a8..9a74537b34 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1980,6 +1980,27 @@ mod tests { driver_mounts::DEFAULT_WORKSPACE_ROOT, ] ); + let custom_image = resolved_image("sha256:image", "1000:1001", "/workspace/project"); + let custom_specs = build_isolation_specs(IsolationSpecInput { + sandbox: &sandbox, + config: &config, + token_secret: Some("jwt"), + resolver_secret: "resolver", + gpu_devices: None, + requested_image: "image:latest", + image: &custom_image, + supervisor_bin: None, + tls_secrets: None, + identity: &identity, + rootless: true, + }) + .unwrap(); + assert_eq!(custom_specs.workload.user, "1000:1001"); + assert!(custom_specs.workload.cap_add.is_empty()); + assert_eq!( + custom_specs.workload.command, + vec!["--bootstrap", crate::isolation::BOOTSTRAP_PATH] + ); let workload_json = serde_json::to_string(&specs.workload).unwrap(); assert!(workload_json.contains("\"apparmor_profile\":\"openshell-sandbox\"")); assert_eq!(specs.supervisor.healthconfig.test, vec!["NONE"]); @@ -2058,137 +2079,22 @@ mod tests { ); } - #[test] - fn custom_workspace_stays_on_the_final_unprivileged_runtime_path() { - let sandbox = DriverSandbox { - id: "pair".into(), - name: "agent".into(), - ..Default::default() - }; - let config = PodmanComputeConfig::default(); - let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( - 1000, - 1001, - vec![2000], - "default".into(), - "sha256:image".into(), - ) - .unwrap(); - let image = resolved_image("sha256:image", "", "/workspace/project"); - - let specs = build_isolation_specs(IsolationSpecInput { - sandbox: &sandbox, - config: &config, - token_secret: None, - resolver_secret: "resolver", - gpu_devices: None, - requested_image: "image:latest", - image: &image, - supervisor_bin: None, - tls_secrets: None, - identity: &identity, - rootless: true, - }) - .unwrap(); - - assert_eq!(specs.workload.work_dir, "/"); - assert_eq!(specs.workload.user, "1000:1001"); - assert!(specs.workload.cap_add.is_empty()); - assert_eq!( - specs.workload.command, - vec!["--bootstrap", crate::isolation::BOOTSTRAP_PATH] - ); - assert!(specs.workload.volumes.iter().any(|volume| { - volume.name == volume_name("pair") && volume.dest == "/workspace/project" - })); - assert_eq!(specs.supervisor.work_dir, "/"); - assert_eq!( - &specs.supervisor.command[..2], - ["--workdir", "/workspace/project"] - ); - assert!( - specs - .supervisor - .volumes - .iter() - .all(|volume| volume.name != volume_name("pair")) - ); - } - - #[test] - fn resolved_image_applies_fallback_and_rejects_unsafe_workdirs() { - for working_dir in ["", "/", "/sandbox"] { - let image = resolved_image("sha256:image", "1000:1000", working_dir); - assert_eq!(image.workspace_root, "/sandbox"); - assert!(image.uses_managed_workspace()); - } - - for working_dir in [ - "relative/workspace", - "/workspace/../project", - "/proc/project", - "/.openshell", - "/.openshell/channel/project", - ] { - let inspected = ImageInspect { - id: "sha256:image".into(), - config: Some(ImageConfig { - user: "1000:1000".into(), - env: Vec::new(), - working_dir: working_dir.into(), - volumes: None, - }), - }; - assert!( - ResolvedPodmanImage::from_inspect(&inspected).is_err(), - "unsafe workdir {working_dir} should be rejected" - ); - } - - let image = resolved_image("sha256:image", "1000:1000", "/usr/src/app"); - assert_eq!(image.workspace_root, "/usr/src/app"); - assert!(!image.uses_managed_workspace()); - } - #[test] fn resolved_image_rejects_masking_oci_volumes_but_allows_nested_volumes() { - for volume in [ - "/workspace", - "/workspace/project", - "/.openshell", - "/.openshell/channel", - ] { - let inspected = ImageInspect { - id: "sha256:image".into(), - config: Some(ImageConfig { - user: "1000:1000".into(), - env: Vec::new(), - working_dir: "/workspace/project".into(), - volumes: Some(std::collections::HashMap::from([( - volume.to_string(), - Value::Object(serde_json::Map::default()), - )])), - }), - }; - assert!( - ResolvedPodmanImage::from_inspect(&inspected).is_err(), - "masking image volume {volume} should be rejected" - ); - } - - let inspected = ImageInspect { + let inspect = |volume: &str| ImageInspect { id: "sha256:image".into(), config: Some(ImageConfig { - user: "1000:1000".into(), - env: Vec::new(), working_dir: "/workspace/project".into(), volumes: Some(std::collections::HashMap::from([( - "/workspace/project/cache".into(), - Value::Object(serde_json::Map::default()), + volume.into(), + Value::Null, )])), + ..Default::default() }), }; - let image = ResolvedPodmanImage::from_inspect(&inspected) + + assert!(ResolvedPodmanImage::from_inspect(&inspect("/workspace")).is_err()); + let image = ResolvedPodmanImage::from_inspect(&inspect("/workspace/project/cache")) .expect("image volumes nested below the workspace remain valid"); assert_eq!(image.workspace_root, "/workspace/project"); } @@ -3284,55 +3190,21 @@ mod tests { } #[test] - fn resolved_workspace_rejects_covering_mounts_but_allows_nested_mounts() { + fn resolved_workspace_rejects_masking_driver_mount() { use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; let image = resolved_image("sha256:immutable", "1000:1000", "/workspace/project"); - for target in ["/workspace", "/workspace/project"] { - let mut sandbox = test_sandbox("test-id", "test-name"); - sandbox.spec = Some(DriverSandboxSpec { - template: Some(DriverSandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "mounts": [{"type": "tmpfs", "target": target}] - }))), - ..Default::default() - }), - ..Default::default() - }); - - let error = build_container_spec_for_image( - &sandbox, - &test_config(), - None, - None, - "image:latest", - &image, - None, - None, - ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("reserved for the OpenShell workspace"), - "covering target {target} should be rejected: {error}" - ); - } - let mut sandbox = test_sandbox("test-id", "test-name"); sandbox.spec = Some(DriverSandboxSpec { template: Some(DriverSandboxTemplate { driver_config: Some(json_struct(serde_json::json!({ - "mounts": [ - {"type": "tmpfs", "target": "/workspace/project/cache"}, - {"type": "tmpfs", "target": "/sandbox"} - ] + "mounts": [{"type": "tmpfs", "target": "/workspace"}] }))), ..Default::default() }), ..Default::default() }); - let spec = build_container_spec_for_image( + let error = build_container_spec_for_image( &sandbox, &test_config(), None, @@ -3342,17 +3214,11 @@ mod tests { None, None, ) - .expect("nested and unrelated mounts should remain valid"); - let mounts = spec["mounts"].as_array().unwrap(); - assert!( - mounts - .iter() - .any(|mount| mount["destination"] == "/workspace/project/cache") - ); + .unwrap_err(); assert!( - mounts - .iter() - .any(|mount| mount["destination"] == "/sandbox") + error + .to_string() + .contains("reserved for the OpenShell workspace") ); } diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 0cf5d65e8b..6e32a211ee 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -269,16 +269,11 @@ Set `process.run_as_user` and `process.run_as_group` in the sandbox policy to ch On Docker and Podman, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR`, `WORKDIR /`, or `WORKDIR /sandbox` use OpenShell's managed -`/sandbox` compatibility workspace. Any other value must be a normalized -absolute path outside kernel and OpenShell control mounts. Driver-config mounts -and image-declared volumes cannot replace that root, but mounts below it remain -valid. +`/sandbox` workspace. Custom paths must be normalized absolute paths outside +runtime and control mounts. Image and driver mounts cannot cover the workspace. Docker validates the directory in the image filesystem. Podman mounts the -persistent workspace volume at the resolved path and preserves Podman's normal -first-use copy-up. OpenShell does not chown, chmod, or otherwise repair a custom -Podman workspace. The final non-root identity must be able to traverse and -write the copied-up directory before the workload starts. Copy-up ownership can -vary with rootless or rootful operation, user namespaces, the backing -filesystem, and SELinux, so validate custom images in the target Podman -environment. Kubernetes and MicroVM continue to use `/sandbox`. +persistent workspace volume at that path with normal image copy-up. OpenShell +preserves its ownership and mode, and fails unless the final non-root identity +can traverse and write it. Validate custom images with the target Podman +configuration. Kubernetes and MicroVM continue to use `/sandbox`. diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index b492c8ac86..2c431a107f 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -56,7 +56,7 @@ required-features = ["e2e-vm"] [[test]] name = "custom_image" path = "tests/custom_image.rs" -required-features = ["e2e-docker"] +required-features = ["e2e-local-container-driver"] [[test]] name = "rootfs_tar" diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs index e2412f8ddf..1618290f1d 100644 --- a/e2e/rust/tests/custom_image.rs +++ b/e2e/rust/tests/custom_image.rs @@ -49,7 +49,6 @@ USER 2345:2346 CMD ["sleep", "infinity"] "#; -#[cfg(feature = "e2e-docker")] const UNWRITABLE_WORKDIR_DOCKERFILE_CONTENT: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ @@ -224,7 +223,6 @@ async fn sandbox_from_passwd_less_numeric_oci_user() { #[tokio::test] #[serial(custom_image)] -#[cfg(feature = "e2e-docker")] async fn sandbox_rejects_image_workdir_that_would_require_new_authority() { let tmpdir = tempfile::tempdir().expect("create tmpdir"); let dockerfile_path = tmpdir.path().join("Dockerfile"); @@ -245,7 +243,7 @@ async fn sandbox_rejects_image_workdir_that_would_require_new_authority() { } Err(error) => error, }; - let message = error.to_string(); + let message = error; assert!( (message.contains("WorkspaceValidationFailed") && message.contains("WorkingDir")) || message.contains("subsystem request failed") diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index f76f430a27..5454401851 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -98,72 +98,6 @@ USER {OCI_UID}:{OCI_GID} "Podman-built image has OCI user '{user}', expected {OCI_UID}:{OCI_GID}" )); } - let working_dir = run_engine( - &engine, - &[ - "image", - "inspect", - "--format", - "{{.Config.WorkingDir}}", - &tag, - ], - )?; - if working_dir != "/home/app/project" { - return Err(format!( - "Podman-built image has OCI workdir '{working_dir}', expected /home/app/project" - )); - } - - Ok(Self { engine, tag, id }) - } - - fn build_unwritable() -> Result { - let engine = ContainerEngine::from_env()?; - if engine.name() != "podman" { - return Err(format!( - "Podman OCI workspace E2E requires podman, got {}", - engine.name() - )); - } - - let context = tempfile::tempdir().map_err(|err| format!("create build context: {err}"))?; - let containerfile = context.path().join("Containerfile"); - std::fs::write( - &containerfile, - format!( - r"FROM {BASE_IMAGE} -USER 0:0 -RUN mkdir -p /root-owned/project && chmod 0700 /root-owned /root-owned/project -WORKDIR /root-owned/project -USER {OCI_UID}:{OCI_GID} -" - ), - ) - .map_err(|err| format!("write Containerfile: {err}"))?; - - let tag = format!( - "localhost/openshell-e2e-podman-unwritable-workdir:{}", - std::process::id() - ); - run_engine( - &engine, - &[ - "build", - "--pull=never", - "--file", - containerfile - .to_str() - .ok_or_else(|| "Containerfile path is not UTF-8".to_string())?, - "--tag", - &tag, - context - .path() - .to_str() - .ok_or_else(|| "build context path is not UTF-8".to_string())?, - ], - )?; - let id = run_engine(&engine, &["image", "inspect", "--format", "{{.Id}}", &tag])?; - Ok(Self { engine, tag, id }) } } @@ -270,9 +204,6 @@ async fn podman_uses_oci_identity_workspace_copy_up_and_inspected_image_id() { "set -eu; \ test \"$(pwd -P)\" = /home/app/project; \ test \"$HOME\" = /home/app/project; \ - test \"$(cat root-owned.txt)\" = root-owned; \ - test \"$(stat -c %u:%g .)\" = 2345:2346; \ - test \"$(stat -c %a .)\" = 700; \ test \"$(stat -c %u:%g root-owned.txt)\" = 0:0; \ touch direct-workspace-write; \ printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; \ @@ -296,7 +227,6 @@ async fn podman_uses_oci_identity_workspace_copy_up_and_inspected_image_id() { "set -eu; \ test \"$(id -u):$(id -g)\" = 2345:2346; \ test \"$(pwd -P)\" = /home/app/project; \ - test \"$HOME\" = /home/app/project; \ test -f direct-workspace-write; \ touch ssh-workspace-write; \ echo podman-ssh-identity-ok", @@ -331,39 +261,6 @@ async fn podman_uses_oci_identity_workspace_copy_up_and_inspected_image_id() { sandbox.cleanup().await; } -#[tokio::test] -async fn podman_rejects_copied_workspace_unusable_by_final_identity() { - if !is_e2e_driver("podman") { - eprintln!("Skipping Podman OCI workspace rejection test: e2e driver is not podman"); - return; - } - - let image = ImageGuard::build_unwritable().expect("build unwritable Podman OCI image"); - let policy = tempfile::NamedTempFile::new().expect("create OCI fallback policy"); - std::fs::write(policy.path(), OCI_FALLBACK_POLICY).expect("write OCI fallback policy"); - let policy_path = policy.path().to_str().expect("policy path is UTF-8"); - let result = SandboxGuard::create_keep_with_args( - &["--from", &image.tag, "--policy", policy_path, "--no-tty"], - &["sh", "-c", "echo should-not-run"], - "should-not-run", - ) - .await; - let error = match result { - Ok(mut sandbox) => { - sandbox.cleanup().await; - panic!("a copied workspace unusable by the final identity must fail closed"); - } - Err(error) => error, - }; - let message = error; - assert!( - (message.contains("WorkspaceValidationFailed") && message.contains("WorkingDir")) - || message.contains("subsystem request failed") - || message.contains("image workspace validation failed"), - "expected copied workspace validation failure, got: {message}" - ); -} - async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, container_id: &str) { let supervisor_id = container_id_for_role(&image.engine, &sandbox.name, "supervisor") .expect("find separate supervisor companion"); diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 15f6af2225..b55a8cdcdd 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -326,13 +326,6 @@ Common findings: - Rootless networking unavailable: inspect Podman network configuration. - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. -- A custom OCI `WORKDIR` uses the Podman workspace volume at that path. Inspect - the workload container's mounts and the copied directory ownership/mode. The - custom path starts directly as the final non-root identity; only managed - `/sandbox` uses OpenShell's root/chown bootstrap. Rootless/rootful services, - user namespaces, backing filesystems, and SELinux can produce different - copy-up results. Fix the image or runtime configuration instead of adding - capabilities or asking OpenShell to repair the custom path. - Supervisor cannot connect: check its gateway endpoint and gateway logs. - Inspect both Podman containers for the sandbox: the `sandbox` isolation role must have network mode `none`; the `supervisor` role owns the gateway session diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index 40149a3b0a..ce116c4797 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -680,11 +680,9 @@ files, mounted volumes, or devices. Docker and Podman gateways also use a normalized absolute OCI `WORKDIR` as the workspace. Empty, `/`, and explicit `/sandbox` declarations use the managed -`/sandbox` fallback. Podman mounts its persistent workspace volume at a custom -workdir and preserves normal first-use image copy-up; it does not repair the -copied ownership or mode. Make the final non-root identity able to traverse and -write that directory, and test the image with the target rootless/rootful, -user-namespace, filesystem, and SELinux configuration. +`/sandbox` fallback. Podman preserves normal volume copy-up and does not repair +custom workspace ownership or mode; the final identity must be able to traverse +and write the directory. ### Forward ports From b95b66a65010f30cf8b757188dcab392338a6e6d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 15:48:06 -0700 Subject: [PATCH 03/20] docs(podman): clarify workspace and image delivery Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 20 ++++++----- crates/openshell-driver-podman/README.md | 35 +++++++++---------- .../openshell-driver-podman/src/container.rs | 5 +++ docs/how-it-works/sandboxes/runtimes.mdx | 15 ++++---- 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 1cbc6088d0..dfd03c9061 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -392,7 +392,7 @@ Drivers deliver the two binaries to separate trust domains: | Runtime | Delivery model | |---|---| | Docker | A digest-pinned daemon-local volume supplies `openshell-sandbox`; the companion image runs `openshell-supervisor`. | -| Podman | A pinned runtime image supplies `openshell-sandbox`; a separate pinned companion image runs `openshell-supervisor`. | +| Podman | The driver pins `sandbox_runtime_image` and `supervisor_image` to image IDs. The former supplies `openshell-sandbox`; the latter is passed as the companion container's image and runs `openshell-supervisor`. | | Kubernetes | A non-root init container stages `openshell-sandbox` into a memory volume; a directly managed Pod runs `openshell-supervisor`. | | VM | `openshell-sandbox` is embedded in the guest rootfs; a separately digest-checked native `openshell-supervisor` runs on the host. | | Extension | Defined by the out-of-tree driver. | @@ -429,14 +429,16 @@ live. Docker and Podman resolve OCI `Config.User` and `Config.WorkingDir` from one immutable image inspection. Empty, `/`, and explicit `/sandbox` values use the managed `/sandbox` workspace. Custom paths must be normalized absolute paths -outside runtime and control mounts; image and driver mounts cannot cover them. - -Podman mounts its persistent workspace volume at a custom root with normal -image copy-up, preserves the resulting ownership and mode, and starts directly -as the final non-root identity. An unusable path fails closed. Only the managed -`/sandbox` fallback uses the root/chown bootstrap. The separate supervisor uses -the path logically but does not mount the workspace. Kubernetes and VM continue -to use `/sandbox`. +that do not overlap `/proc`, `/sys`, `/dev`, or OpenShell's private paths. Image +and driver mounts cannot cover the workspace path or one of its parents. + +Podman mounts its persistent workspace volume at a custom root. When the volume +is first created, Podman copies existing image-directory contents into it. +OpenShell preserves their ownership and mode and starts directly as the final +non-root identity. An unusable path fails closed. Only the managed `/sandbox` +fallback uses the root/chown bootstrap. The separate supervisor receives the +path but does not mount the workspace. Kubernetes and VM continue to use +`/sandbox`. ### Executable Identity Binding diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 63634ed1a9..bd86d8aee1 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -92,24 +92,23 @@ environment belong to agent children, never the supervisor process. ## OCI working directory -The immutable workload-image inspection also resolves OCI `WorkingDir`. Empty, -`/`, and explicit `/sandbox` values select the managed `/sandbox` workspace. -Custom paths must be normalized absolute paths outside runtime and control -mounts. Image and driver mounts may be nested below the workspace but cannot -cover it. - -For a custom root, the driver mounts the persistent workspace volume at the -resolved path without `nocopy` or ownership-changing volume options. Podman -performs its normal first-use copy-up from the image. OpenShell preserves the -copied ownership and mode, starts directly as the final non-root identity, and -fails closed unless that identity can traverse and write the path. Agent -children use the path as cwd and `HOME`; `filesystem.include_workdir` grants it -when enabled. - -Only `/sandbox` uses the root-to-non-root bootstrap needed to prepare a managed -workspace. The separate supervisor receives the path logically but does not -mount it. Because Podman copy-up varies by deployment, validate custom images -with the target runtime configuration. +OpenShell reads `WORKDIR` from the workload image. If it is unset, `/`, or +`/sandbox`, OpenShell uses its managed `/sandbox` workspace. A custom path must +be absolute, with no `.` or `..` segments. It cannot overlap container system +paths (`/proc`, `/sys`, `/dev`) or OpenShell's private paths (for example, +`/.openshell` and `/run/openshell`). Image and driver mounts may be inside the +workspace, but cannot replace the workspace path or one of its parents. + +For a custom path, Podman mounts a persistent workspace volume there. When the +volume is first created, Podman copies any files already in that image directory +into it. OpenShell keeps their ownership and permissions, starts as the final +non-root user, and rejects the image if that user cannot reach and write the +directory. Agent commands use the path as their working directory and `HOME`; +`filesystem.include_workdir` grants access to it when enabled. + +For `/sandbox`, OpenShell prepares the managed workspace before switching to +the non-root user. The separate supervisor receives the path but does not mount +the workspace. Test custom images with the Podman configuration you will use. ## Lifecycle and readiness diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 9a74537b34..1820902f53 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -228,6 +228,11 @@ pub struct ResolvedPodmanImage { } impl ResolvedPodmanImage { + /// Resolve the image metadata and reject: + /// - a malformed or relative working directory, or one overlapping runtime + /// or `OpenShell` control paths; + /// - an image volume with an invalid or reserved target; and + /// - an image volume covering the workspace or any of its ancestors. pub fn from_inspect(inspected: &ImageInspect) -> Result { let image_config = inspected.config.as_ref(); let workspace_root = driver_mounts::resolve_oci_workspace_root( diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 6e32a211ee..ef546ec550 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -269,11 +269,14 @@ Set `process.run_as_user` and `process.run_as_group` in the sandbox policy to ch On Docker and Podman, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR`, `WORKDIR /`, or `WORKDIR /sandbox` use OpenShell's managed -`/sandbox` workspace. Custom paths must be normalized absolute paths outside -runtime and control mounts. Image and driver mounts cannot cover the workspace. +`/sandbox` workspace. Custom paths must be absolute, with no `.` or `..` +segments, and cannot overlap container system paths (`/proc`, `/sys`, `/dev`) +or OpenShell's private paths. Image and driver mounts cannot replace the +workspace or one of its parents. Docker validates the directory in the image filesystem. Podman mounts the -persistent workspace volume at that path with normal image copy-up. OpenShell -preserves its ownership and mode, and fails unless the final non-root identity -can traverse and write it. Validate custom images with the target Podman -configuration. Kubernetes and MicroVM continue to use `/sandbox`. +persistent workspace volume at that path. When the volume is first created, +Podman copies existing files from the image directory into it. OpenShell keeps +their ownership and permissions, and fails unless the final non-root user can +reach and write the directory. Test custom images with the Podman configuration +you will use. Kubernetes and MicroVM continue to use `/sandbox`. From 0d20c7ffe56eba760cd47aeb58b51189855056f5 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 16:58:21 -0700 Subject: [PATCH 04/20] fix(drivers): narrow OCI workspace path restrictions Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 5 +- crates/openshell-core/src/driver_mounts.rs | 122 +++++------------- crates/openshell-driver-docker/README.md | 6 +- crates/openshell-driver-docker/src/lib.rs | 40 ++++-- crates/openshell-driver-docker/src/tests.rs | 30 ++++- crates/openshell-driver-podman/README.md | 11 +- .../openshell-driver-podman/src/container.rs | 79 ++++++++++-- docs/how-it-works/sandboxes/runtimes.mdx | 3 +- skills/debug-openshell-cluster/SKILL.md | 2 +- 9 files changed, 171 insertions(+), 127 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index dfd03c9061..0209110804 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -429,8 +429,9 @@ live. Docker and Podman resolve OCI `Config.User` and `Config.WorkingDir` from one immutable image inspection. Empty, `/`, and explicit `/sandbox` values use the managed `/sandbox` workspace. Custom paths must be normalized absolute paths -that do not overlap `/proc`, `/sys`, `/dev`, or OpenShell's private paths. Image -and driver mounts cannot cover the workspace path or one of its parents. +that do not overlap `/proc`, `/sys`, `/dev`, or private mounts still inside the +workload container. Image and driver mounts cannot cover the workspace path or +one of its parents. Supervisor-only paths are not reserved in the workload. Podman mounts its persistent workspace volume at a custom root. When the volume is first created, Podman copies existing image-directory contents into it. diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index b1a3049882..a97f666cc0 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -84,9 +84,17 @@ pub fn validate_mount_subpath(subpath: &str) -> Result<(), String> { /// Workspace collisions depend on the inspected image's resolved working /// directory and are checked separately by `validate_workspace_mount_target`. pub fn validate_container_mount_target(target: &str) -> Result<(), String> { + validate_container_mount_target_for_workload(target, CONTROL_ROOTS) +} + +/// Validate a mount target against paths used by this specific workload. +pub fn validate_container_mount_target_for_workload( + target: &str, + workload_reserved_paths: &[&str], +) -> Result<(), String> { let normalized = normalize_absolute_container_path(target, "mount target")?; let path = Path::new(&normalized); - for reserved in CONTROL_ROOTS { + for reserved in workload_reserved_paths { let reserved = Path::new(reserved); if paths_overlap(path, reserved) { return Err(format!( @@ -106,6 +114,16 @@ pub fn validate_container_mount_target(target: &str) -> Result<(), String> { /// value and the path passed to the supervisor cannot be interpreted /// differently. pub fn resolve_oci_workspace_root(working_dir: &str) -> Result { + // The sandbox runtime checks syntax and OCI mounts again; each compute + // driver checks its own workload mounts before admitting the workspace. + resolve_oci_workspace_root_for_workload(working_dir, &[]) +} + +/// Resolve a workspace against paths still mounted inside this workload. +pub fn resolve_oci_workspace_root_for_workload( + working_dir: &str, + workload_reserved_paths: &[&str], +) -> Result { if working_dir.is_empty() || working_dir == "/" { return Ok(DEFAULT_WORKSPACE_ROOT.to_string()); } @@ -113,7 +131,7 @@ pub fn resolve_oci_workspace_root(working_dir: &str) -> Result { for runtime_path in OCI_RUNTIME_MOUNT_ROOTS { validate_workspace_reserved_path(&workspace_root, runtime_path, "OCI runtime mount")?; } - for control_path in CONTROL_ROOTS { + for control_path in workload_reserved_paths { validate_workspace_control_path(&workspace_root, control_path)?; } @@ -178,23 +196,6 @@ fn validate_workspace_reserved_path( Ok(()) } -/// Reject a mount that contains or is contained by a runtime-configured -/// `OpenShell` control path, such as the sandbox SSH socket. -pub fn validate_mount_control_path(target: &str, control_path: &str) -> Result<(), String> { - let normalized_target = normalize_absolute_container_path(target, "mount target")?; - let normalized_control = - normalize_absolute_container_path(control_path, "OpenShell control path")?; - if paths_overlap( - Path::new(&normalized_target), - Path::new(&normalized_control), - ) { - return Err(format!( - "mount target '{target}' conflicts with OpenShell control path '{control_path}'" - )); - } - Ok(()) -} - /// Reject a user-supplied mount that would replace or contain the resolved /// workspace root. Mounts below the workspace remain valid. pub fn validate_workspace_mount_target(target: &str, workspace_root: &str) -> Result<(), String> { @@ -278,86 +279,33 @@ mod tests { } #[test] - fn oci_workspace_root_rejects_runtime_and_openshell_control_path_collisions() { + fn oci_workspace_root_rejects_runtime_and_selected_workload_paths() { + let reserved = &["/control"]; for invalid in [ "/proc", "/proc/self", "/sys", - "/sys/fs/cgroup", - "/dev", "/dev/shm", - "/etc", - "/opt", - "/opt/openshell", - "/opt/openshell/bin/project", - "/etc/openshell/tls/client", - "/etc/openshell/auth", - "/etc/openshell/skills", - "/etc/openshell-tls", - "/run", - "/run/openshell/cache", - "/run/openshell-sidecar/control.sock", - "/run/netns/project", - "/var/run/netns/project", + "/control", + "/control/data", ] { assert!( - resolve_oci_workspace_root(invalid).is_err(), - "expected control-path workspace '{invalid}' to be rejected" - ); - } - - for valid in [ - "/app", - "/etc/project", - "/home/app", - "/opt/app", - "/usr/bin/project", - "/usr/src/app", - "/var/lib/app", - "/var/app/current", - "/var/task", - "/var/www/app", - "/processor", - "/system", - "/device", - ] { - assert_eq!( - resolve_oci_workspace_root(valid).unwrap(), - valid, - "expected application workspace '{valid}' to remain valid" + resolve_oci_workspace_root_for_workload(invalid, reserved).is_err(), + "expected workspace '{invalid}' to be rejected" ); } + assert_eq!( + resolve_oci_workspace_root_for_workload("/etc/openshell", reserved).unwrap(), + "/etc/openshell" + ); } #[test] - fn container_target_rejects_reserved_openshell_tls_legacy_path() { - let err = validate_container_mount_target("/etc/openshell-tls/proxy/client").unwrap_err(); - - assert!(err.contains("/etc/openshell-tls")); - } - - #[test] - fn container_target_rejects_reserved_openshell_tree() { - let err = validate_container_mount_target("/etc/openshell/tls/client").unwrap_err(); - - assert!(err.contains("/etc/openshell")); - } - - #[test] - fn container_target_does_not_prefix_match_unrelated_paths() { - validate_container_mount_target("/etc/openshell-tools").unwrap(); - validate_container_mount_target("/run/openshell-tools").unwrap(); - } - - #[test] - fn mount_target_rejects_runtime_configured_control_path_overlap() { - for target in ["/custom", "/custom/ssh.sock", "/custom/ssh.sock/cache"] { - assert!( - validate_mount_control_path(target, "/custom/ssh.sock").is_err(), - "expected '{target}' to conflict with the configured control path" - ); - } - validate_mount_control_path("/custom-other", "/custom/ssh.sock").unwrap(); + fn container_target_uses_selected_workload_paths() { + let reserved = &["/control"]; + assert!(validate_container_mount_target_for_workload("/control/data", reserved).is_err()); + validate_container_mount_target_for_workload("/control-tools", reserved).unwrap(); + validate_container_mount_target_for_workload("/etc/openshell", reserved).unwrap(); } #[test] diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index a25c455eea..d40f86c00c 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -67,7 +67,8 @@ traverse every parent and write and enter the workdir; OpenShell does not change its ownership or mode. Image `VOLUME` declarations and user mounts must not cover the workdir, one of -its parents, or the reserved `/.openshell` runtime/channel tree. OpenShell asks +its parents, the workload's `/.openshell` runtime/channel tree, or its +`/run/openshell-supervisor-ca` mount. OpenShell asks the kernel to validate access under the final identity, so POSIX ACL and host LSM decisions remain authoritative. @@ -114,7 +115,8 @@ mount types are: Host bind mounts are disabled by default because they expose daemon-host paths to sandbox requests. User bind and volume mounts are read-only by default. Targets must be absolute, normalized paths and cannot overlap the workspace -root or OpenShell control paths. +root or private mounts still used inside the workload. Supervisor-only paths +are allowed. Example: diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 1f4c91f5e4..ed5e2c01cf 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -105,6 +105,10 @@ const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/.openshell/channel/sandbox/bootstrap. const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/sandbox/control.sock"; const BOUNDARY_CERTIFICATE_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.crt"; const BOUNDARY_PRIVATE_KEY_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.key"; +const WORKLOAD_RESERVED_PATHS: &[&str] = &[ + "/.openshell", + openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, +]; const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/supervisor"; const SUPERVISOR_PROXY_AUTH_MOUNT_PATH: &str = "/.openshell/supervisor/upstream-proxy-auth"; const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = @@ -3759,7 +3763,8 @@ fn docker_bind_string( "bind source path does not exist: {source}" ))); } - driver_mounts::validate_container_mount_target(target).map_err(Status::failed_precondition)?; + driver_mounts::validate_container_mount_target_for_workload(target, WORKLOAD_RESERVED_PATHS) + .map_err(Status::failed_precondition)?; let normalized_target = driver_mounts::normalize_mount_target(target); let mut opts = Vec::new(); @@ -3897,8 +3902,11 @@ fn validate_docker_driver_mounts( )); } }; - driver_mounts::validate_container_mount_target(target) - .map_err(Status::failed_precondition)?; + driver_mounts::validate_container_mount_target_for_workload( + target, + WORKLOAD_RESERVED_PATHS, + ) + .map_err(Status::failed_precondition)?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(Status::failed_precondition(format!( @@ -4524,8 +4532,11 @@ async fn prepare_docker_boundary_files( gpu_requested: bool, ) -> Result<(), Status> { let directory = docker_boundary_state_dir(sandbox, config)?; - let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) - .map_err(Status::failed_precondition)?; + let workspace_root = driver_mounts::resolve_oci_workspace_root_for_workload( + &image.working_dir, + WORKLOAD_RESERVED_PATHS, + ) + .map_err(Status::failed_precondition)?; let launch_authentication = sandbox .spec .as_ref() @@ -5652,12 +5663,17 @@ fn build_container_create_body_for_image( .as_ref() .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; let resource_limits = docker_resource_limits(template)?; - let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) - .map_err(Status::failed_precondition)?; - driver_mounts::validate_workspace_control_path(&workspace_root, BOUNDARY_MOUNT_PATH) - .map_err(Status::failed_precondition)?; + let workspace_root = driver_mounts::resolve_oci_workspace_root_for_workload( + &image.working_dir, + WORKLOAD_RESERVED_PATHS, + ) + .map_err(Status::failed_precondition)?; for volume in &image.volumes { - driver_mounts::validate_container_mount_target(volume).map_err(|error| { + driver_mounts::validate_container_mount_target_for_workload( + volume, + WORKLOAD_RESERVED_PATHS, + ) + .map_err(|error| { Status::failed_precondition(format!( "invalid image-declared volume '{volume}': {error}" )) @@ -5667,8 +5683,6 @@ fn build_container_create_body_for_image( "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" )) })?; - driver_mounts::validate_mount_control_path(volume, BOUNDARY_MOUNT_PATH) - .map_err(Status::failed_precondition)?; } for mount in &driver_config.mounts { let target = match mount { @@ -5679,8 +5693,6 @@ fn build_container_create_body_for_image( }; driver_mounts::validate_workspace_mount_target(target, &workspace_root) .map_err(Status::failed_precondition)?; - driver_mounts::validate_mount_control_path(target, BOUNDARY_MOUNT_PATH) - .map_err(Status::failed_precondition)?; } let mut user_mounts = docker_driver_mounts(driver_config)?; user_mounts.push(Mount { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index c5036f4547..811ebd6fd1 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -1551,10 +1551,10 @@ fn container_creation_rejects_invalid_oci_working_dir() { #[test] fn container_creation_rejects_openshell_control_path_working_dir() { - let metadata = DockerImageMetadata { + let mut metadata = DockerImageMetadata { id: "sha256:immutable".to_string(), user: "1234:1235".to_string(), - working_dir: "/opt/openshell/bin/project".to_string(), + working_dir: "/.openshell/runtime/project".to_string(), volumes: Vec::new(), }; let err = build_container_create_body_for_image( @@ -1569,6 +1569,17 @@ fn container_creation_rejects_openshell_control_path_working_dir() { assert_eq!(err.code(), tonic::Code::FailedPrecondition); assert!(err.message().contains("OpenShell control path")); + + metadata.working_dir = "/opt/openshell/bin/project".to_string(); + build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &DockerSandboxDriverConfig::default(), + None, + &metadata, + &test_workload_identity(), + ) + .expect("supervisor-only paths are not reserved in the workload"); } #[test] @@ -2145,7 +2156,7 @@ fn driver_config_rejects_reserved_mount_targets() { "mounts": [{ "type": "volume", "source": "work-nfs", - "target": "/etc/openshell/auth" + "target": "/.openshell/runtime" }] }))); @@ -2153,6 +2164,19 @@ fn driver_config_rejects_reserved_mount_targets() { assert_eq!(err.code(), tonic::Code::FailedPrecondition); assert!(err.message().contains("reserved OpenShell path")); + + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(json_struct(serde_json::json!({ + "mounts": [{"type": "volume", "source": "work-nfs", "target": "/etc/openshell/auth"}] + }))); + build_container_create_body(&sandbox, &runtime_config()) + .expect("supervisor-only paths are not reserved in the workload"); } #[test] diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index bd86d8aee1..d621001527 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -95,9 +95,12 @@ environment belong to agent children, never the supervisor process. OpenShell reads `WORKDIR` from the workload image. If it is unset, `/`, or `/sandbox`, OpenShell uses its managed `/sandbox` workspace. A custom path must be absolute, with no `.` or `..` segments. It cannot overlap container system -paths (`/proc`, `/sys`, `/dev`) or OpenShell's private paths (for example, -`/.openshell` and `/run/openshell`). Image and driver mounts may be inside the -workspace, but cannot replace the workspace path or one of its parents. +paths (`/proc`, `/sys`, `/dev`) or mounts still used in the workload: +`/.openshell` for the control channel, `/opt/openshell/bin` for the sandbox +runtime, and `/run/openshell-supervisor-ca` for generated CA material. Paths +used only by the separate supervisor are allowed. Image and driver mounts may +be inside the workspace, but cannot replace the workspace path or one of its +parents. For a custom path, Podman mounts a persistent workspace volume there. When the volume is first created, Podman copies any files already in that image directory @@ -136,7 +139,7 @@ User `bind`, `volume`, `tmpfs`, and `image` mounts and CDI GPU selection remain native Podman features and apply only to the workload. Bind mounts require the operator's `enable_bind_mounts` opt-in and disabled label admission. Supplemental image mounts also require disabled admission. Driver JSON requires -`allow_driver_config = true`. Reserved control paths and the workspace +`allow_driver_config = true`. The workload's private mounts and workspace root cannot be replaced. User-owned volumes are never created or deleted. See [gateway configuration](../../docs/how-it-works/gateways/configuration.mdx) for diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 1820902f53..f09311ecd6 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -75,6 +75,11 @@ const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = const SUPERVISOR_MOUNT_DIR: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_DIR; /// Full path to the supervisor binary inside sandbox containers. const SUPERVISOR_BINARY_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; +const WORKLOAD_RESERVED_PATHS: &[&str] = &[ + "/.openshell", + SUPERVISOR_MOUNT_DIR, + openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, +]; #[derive(Debug, Clone, Default, serde::Deserialize)] #[serde(default, deny_unknown_fields)] @@ -235,15 +240,18 @@ impl ResolvedPodmanImage { /// - an image volume covering the workspace or any of its ancestors. pub fn from_inspect(inspected: &ImageInspect) -> Result { let image_config = inspected.config.as_ref(); - let workspace_root = driver_mounts::resolve_oci_workspace_root( + let workspace_root = driver_mounts::resolve_oci_workspace_root_for_workload( image_config.map_or("", |config| config.working_dir.as_str()), + WORKLOAD_RESERVED_PATHS, ) .map_err(ComputeDriverError::Precondition)?; - driver_mounts::validate_workspace_control_path(&workspace_root, "/.openshell") - .map_err(ComputeDriverError::Precondition)?; if let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) { for volume in volumes.keys() { - driver_mounts::validate_container_mount_target(volume).map_err(|error| { + driver_mounts::validate_container_mount_target_for_workload( + volume, + WORKLOAD_RESERVED_PATHS, + ) + .map_err(|error| { ComputeDriverError::Precondition(format!( "invalid image-declared volume '{volume}': {error}" )) @@ -255,8 +263,6 @@ impl ResolvedPodmanImage { )) }, )?; - driver_mounts::validate_mount_control_path(volume, "/.openshell") - .map_err(ComputeDriverError::Precondition)?; } } @@ -871,7 +877,10 @@ fn podman_user_mounts( None => {} } driver_mounts::validate_absolute_mount_source(&source, "bind source")?; - driver_mounts::validate_container_mount_target(&target)?; + driver_mounts::validate_container_mount_target_for_workload( + &target, + WORKLOAD_RESERVED_PATHS, + )?; result.mounts.push(Mount { kind: "bind".into(), source, @@ -887,7 +896,10 @@ fn podman_user_mounts( } => { reject_subpath(subpath.as_deref(), "podman volume mounts")?; driver_mounts::validate_mount_source(&source, "volume source")?; - driver_mounts::validate_container_mount_target(&target)?; + driver_mounts::validate_container_mount_target_for_workload( + &target, + WORKLOAD_RESERVED_PATHS, + )?; result.volumes.push(NamedVolume { name: source, dest: target, @@ -913,7 +925,10 @@ fn podman_user_mounts( { options.push(format!("mode={mode:o}")); } - driver_mounts::validate_container_mount_target(&target)?; + driver_mounts::validate_container_mount_target_for_workload( + &target, + WORKLOAD_RESERVED_PATHS, + )?; result.mounts.push(Mount { kind: "tmpfs".into(), source: "tmpfs".into(), @@ -929,7 +944,10 @@ fn podman_user_mounts( } => { reject_subpath(subpath.as_deref(), "podman image mounts")?; driver_mounts::validate_mount_source(&source, "image source")?; - driver_mounts::validate_container_mount_target(&target)?; + driver_mounts::validate_container_mount_target_for_workload( + &target, + WORKLOAD_RESERVED_PATHS, + )?; result.image_volumes.push(ImageVolume { source, destination: target, @@ -1004,8 +1022,10 @@ fn validate_podman_driver_mounts( target } }; - driver_mounts::validate_container_mount_target(target)?; - driver_mounts::validate_mount_control_path(target, "/.openshell")?; + driver_mounts::validate_container_mount_target_for_workload( + target, + WORKLOAD_RESERVED_PATHS, + )?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(format!( @@ -2104,6 +2124,26 @@ mod tests { assert_eq!(image.workspace_root, "/workspace/project"); } + #[test] + fn resolved_image_allows_supervisor_only_workdir_but_reserves_workload_mounts() { + let inspect = |working_dir: &str| ImageInspect { + id: "sha256:image".into(), + config: Some(ImageConfig { + working_dir: working_dir.into(), + ..Default::default() + }), + }; + + assert!(ResolvedPodmanImage::from_inspect(&inspect("/opt/openshell/bin/project")).is_err()); + assert!(ResolvedPodmanImage::from_inspect(&inspect("/.openshell/channel")).is_err()); + assert_eq!( + ResolvedPodmanImage::from_inspect(&inspect("/etc/openshell/tls/client")) + .unwrap() + .workspace_root, + "/etc/openshell/tls/client" + ); + } + fn json_struct(value: Value) -> prost_types::Struct { let Value::Object(object) = value else { panic!("expected JSON object"); @@ -3449,7 +3489,7 @@ mod tests { "mounts": [{ "type": "volume", "source": "work-nfs", - "target": "/etc/openshell/tls/client" + "target": "/opt/openshell/bin" }] }))), ..Default::default() @@ -3461,6 +3501,19 @@ mod tests { let err = try_build_container_spec_with_token(&sandbox, &config, None).unwrap_err(); assert!(err.to_string().contains("reserved OpenShell path")); + + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(json_struct(serde_json::json!({ + "mounts": [{"type": "volume", "source": "work-nfs", "target": "/etc/openshell/tls/client"}] + }))); + try_build_container_spec_with_token(&sandbox, &config, None) + .expect("supervisor-only paths are not reserved in the workload"); } #[test] diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index ef546ec550..08053b50c5 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -271,7 +271,8 @@ On Docker and Podman, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR`, `WORKDIR /`, or `WORKDIR /sandbox` use OpenShell's managed `/sandbox` workspace. Custom paths must be absolute, with no `.` or `..` segments, and cannot overlap container system paths (`/proc`, `/sys`, `/dev`) -or OpenShell's private paths. Image and driver mounts cannot replace the +or private mounts still used inside the workload. Paths used only by the +separate supervisor are allowed. Image and driver mounts cannot replace the workspace or one of its parents. Docker validates the directory in the image filesystem. Podman mounts the diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index b55a8cdcdd..117908e13c 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -281,7 +281,7 @@ Common findings: - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default Docker workdir or a copied-up custom Podman workspace. - Docker and Podman also reject an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the path before validation. Move the `VOLUME` below the workspace or remove the declaration. -- A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. +- A workdir rejected as a special filesystem or workload-mount collision cannot be made valid with permissions. Move it away from `/proc`, `/sys`, `/dev`, and the sandbox runtime or control-channel paths named in the error. Paths used only by the separate supervisor are allowed. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. From cba9b6bb4cdcedc2ed5881ff7ab2282cc4cf6186 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 17:30:27 -0700 Subject: [PATCH 05/20] fix(drivers): allow workload workdir mount overlap Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 7 +- crates/openshell-core/src/container_paths.rs | 29 ----- crates/openshell-core/src/driver_mounts.rs | 33 +++-- crates/openshell-driver-docker/README.md | 17 +-- crates/openshell-driver-docker/src/lib.rs | 17 +-- crates/openshell-driver-docker/src/tests.rs | 115 +++++------------- crates/openshell-driver-podman/README.md | 17 ++- .../openshell-driver-podman/src/container.rs | 46 +++---- docs/how-it-works/sandboxes/runtimes.mdx | 14 ++- skills/debug-openshell-cluster/SKILL.md | 4 +- 10 files changed, 95 insertions(+), 204 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 0209110804..c15ea662bc 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -429,9 +429,10 @@ live. Docker and Podman resolve OCI `Config.User` and `Config.WorkingDir` from one immutable image inspection. Empty, `/`, and explicit `/sandbox` values use the managed `/sandbox` workspace. Custom paths must be normalized absolute paths -that do not overlap `/proc`, `/sys`, `/dev`, or private mounts still inside the -workload container. Image and driver mounts cannot cover the workspace path or -one of its parents. Supervisor-only paths are not reserved in the workload. +that do not overlap private mounts still inside the workload container. +Supervisor-only paths outside that private tree are not reserved. Image and +driver mounts may cover the workspace; conflicting mounts can make the workload +unusable. Private workload mount targets remain reserved. Podman mounts its persistent workspace volume at a custom root. When the volume is first created, Podman copies existing image-directory contents into it. diff --git a/crates/openshell-core/src/container_paths.rs b/crates/openshell-core/src/container_paths.rs index 63511c13ff..1dab75bd0a 100644 --- a/crates/openshell-core/src/container_paths.rs +++ b/crates/openshell-core/src/container_paths.rs @@ -16,15 +16,6 @@ pub const SIDECAR_RUN_ROOT: &str = "/run/openshell-sidecar"; pub const NETNS_MOUNT_ROOT: &str = "/run/netns"; pub const NETNS_IPROUTE2_ROOT: &str = "/var/run/netns"; -/// Standard Linux container namespaces that an image-selected workspace must -/// not contain or enter. -/// -/// These roots cover the default filesystems and devices defined by the OCI -/// Runtime Specification: procfs, sysfs, cgroups, device nodes, devpts, shared -/// memory, and POSIX message queues. -/// -pub const OCI_RUNTIME_MOUNT_ROOTS: &[&str] = &["/proc", "/sys", "/dev"]; - /// High-level namespaces mounted or created by `OpenShell` inside sandboxes. /// /// This is intentionally not a general Linux system-path denylist. Kernel and @@ -144,24 +135,4 @@ mod tests { ); } } - - #[test] - fn runtime_roots_cover_standard_oci_mount_destinations() { - for path in [ - "/proc", - "/dev", - "/dev/pts", - "/dev/shm", - "/dev/mqueue", - "/sys", - "/sys/fs/cgroup", - ] { - assert!( - OCI_RUNTIME_MOUNT_ROOTS - .iter() - .any(|root| Path::new(path).starts_with(root)), - "OCI runtime mount {path} is outside the reserved roots" - ); - } - } } diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index a97f666cc0..7402f778b1 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -5,7 +5,7 @@ use std::path::Path; -use crate::container_paths::{CONTROL_ROOTS, OCI_RUNTIME_MOUNT_ROOTS}; +use crate::container_paths::CONTROL_ROOTS; /// `SELinux` relabelling mode for bind mounts. /// @@ -81,8 +81,7 @@ pub fn validate_mount_subpath(subpath: &str) -> Result<(), String> { /// Validate a container-side mount target for user-supplied driver mounts. /// -/// Workspace collisions depend on the inspected image's resolved working -/// directory and are checked separately by `validate_workspace_mount_target`. +/// Drivers may apply additional checks for mounts used by their workload. pub fn validate_container_mount_target(target: &str) -> Result<(), String> { validate_container_mount_target_for_workload(target, CONTROL_ROOTS) } @@ -114,8 +113,8 @@ pub fn validate_container_mount_target_for_workload( /// value and the path passed to the supervisor cannot be interpreted /// differently. pub fn resolve_oci_workspace_root(working_dir: &str) -> Result { - // The sandbox runtime checks syntax and OCI mounts again; each compute - // driver checks its own workload mounts before admitting the workspace. + // The sandbox runtime checks syntax again; each compute driver checks + // its own workload mounts before admitting the workspace. resolve_oci_workspace_root_for_workload(working_dir, &[]) } @@ -128,9 +127,6 @@ pub fn resolve_oci_workspace_root_for_workload( return Ok(DEFAULT_WORKSPACE_ROOT.to_string()); } let workspace_root = normalize_absolute_container_path(working_dir, "OCI WorkingDir")?; - for runtime_path in OCI_RUNTIME_MOUNT_ROOTS { - validate_workspace_reserved_path(&workspace_root, runtime_path, "OCI runtime mount")?; - } for control_path in workload_reserved_paths { validate_workspace_control_path(&workspace_root, control_path)?; } @@ -196,8 +192,8 @@ fn validate_workspace_reserved_path( Ok(()) } -/// Reject a user-supplied mount that would replace or contain the resolved -/// workspace root. Mounts below the workspace remain valid. +/// Reject a user-supplied mount that would replace or contain a driver-managed +/// workspace root. Kubernetes uses this for its fixed workspace mount. pub fn validate_workspace_mount_target(target: &str, workspace_root: &str) -> Result<(), String> { let normalized_target = normalize_mount_target(target); if path_is_or_under(Path::new(workspace_root), Path::new(&normalized_target)) { @@ -279,16 +275,9 @@ mod tests { } #[test] - fn oci_workspace_root_rejects_runtime_and_selected_workload_paths() { + fn oci_workspace_root_only_reserves_selected_workload_paths() { let reserved = &["/control"]; - for invalid in [ - "/proc", - "/proc/self", - "/sys", - "/dev/shm", - "/control", - "/control/data", - ] { + for invalid in ["/control", "/control/data"] { assert!( resolve_oci_workspace_root_for_workload(invalid, reserved).is_err(), "expected workspace '{invalid}' to be rejected" @@ -298,6 +287,12 @@ mod tests { resolve_oci_workspace_root_for_workload("/etc/openshell", reserved).unwrap(), "/etc/openshell" ); + for path in ["/proc", "/sys", "/dev/shm"] { + assert_eq!( + resolve_oci_workspace_root_for_workload(path, reserved).unwrap(), + path + ); + } } #[test] diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index d40f86c00c..7dde4011d7 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -66,11 +66,12 @@ already exist without symlink components. The resolved identity must be able to traverse every parent and write and enter the workdir; OpenShell does not change its ownership or mode. -Image `VOLUME` declarations and user mounts must not cover the workdir, one of -its parents, the workload's `/.openshell` runtime/channel tree, or its -`/run/openshell-supervisor-ca` mount. OpenShell asks -the kernel to validate access under the final identity, so POSIX ACL and host -LSM decisions remain authoritative. +The workdir cannot overlap the workload's `/.openshell` runtime/channel tree +or its `/run/openshell-supervisor-ca` mount. Image `VOLUME` declarations and +user mounts may cover the workdir; Docker's final mount layout determines +whether that directory is usable. OpenShell asks the kernel to validate access +under the final identity, so POSIX ACL and host LSM decisions remain +authoritative. ## Container Contract @@ -114,9 +115,9 @@ mount types are: Host bind mounts are disabled by default because they expose daemon-host paths to sandbox requests. User bind and volume mounts are read-only by default. -Targets must be absolute, normalized paths and cannot overlap the workspace -root or private mounts still used inside the workload. Supervisor-only paths -are allowed. +Targets must be absolute, normalized paths and cannot overlap private mounts +still used inside the workload. They may overlap the workspace; paths outside +the workload's private tree that are used only by the supervisor are allowed. Example: diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index ed5e2c01cf..ed3245b743 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5663,7 +5663,7 @@ fn build_container_create_body_for_image( .as_ref() .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; let resource_limits = docker_resource_limits(template)?; - let workspace_root = driver_mounts::resolve_oci_workspace_root_for_workload( + driver_mounts::resolve_oci_workspace_root_for_workload( &image.working_dir, WORKLOAD_RESERVED_PATHS, ) @@ -5678,21 +5678,6 @@ fn build_container_create_body_for_image( "invalid image-declared volume '{volume}': {error}" )) })?; - driver_mounts::validate_workspace_mount_target(volume, &workspace_root).map_err(|_| { - Status::failed_precondition(format!( - "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" - )) - })?; - } - for mount in &driver_config.mounts { - let target = match mount { - DockerDriverMountConfig::Bind { target, .. } - | DockerDriverMountConfig::Volume { target, .. } - | DockerDriverMountConfig::Tmpfs { target, .. } - | DockerDriverMountConfig::Image { target, .. } => target, - }; - driver_mounts::validate_workspace_mount_target(target, &workspace_root) - .map_err(Status::failed_precondition)?; } let mut user_mounts = docker_driver_mounts(driver_config)?; user_mounts.push(Mount { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 811ebd6fd1..82443d09ba 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -1580,10 +1580,21 @@ fn container_creation_rejects_openshell_control_path_working_dir() { &test_workload_identity(), ) .expect("supervisor-only paths are not reserved in the workload"); + + metadata.working_dir = "/proc".to_string(); + build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &DockerSandboxDriverConfig::default(), + None, + &metadata, + &test_workload_identity(), + ) + .expect("OCI system paths are not rejected before workload launch"); } #[test] -fn container_creation_rejects_image_volume_that_masks_working_dir() { +fn container_creation_allows_image_volume_covering_working_dir() { let sandbox = test_sandbox(); let metadata = DockerImageMetadata { id: "sha256:immutable".to_string(), @@ -1592,7 +1603,7 @@ fn container_creation_rejects_image_volume_that_masks_working_dir() { volumes: vec!["/workspace".to_string()], }; - let error = build_container_create_body_for_image( + build_container_create_body_for_image( &sandbox, &runtime_config(), &DockerSandboxDriverConfig::default(), @@ -1600,92 +1611,32 @@ fn container_creation_rejects_image_volume_that_masks_working_dir() { &metadata, &test_workload_identity(), ) - .unwrap_err(); - - assert!( - error - .message() - .contains("masks OCI WorkingDir '/workspace/project'") - ); + .expect("image volumes may cover a workload-owned workspace"); } #[test] -fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts() { - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/workspace".to_string(), - volumes: Vec::new(), - }; - let root_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ - "mounts": [{"type": "tmpfs", "target": "/workspace"}] - })) - .unwrap(); - let err = build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &root_mount, - None, - &metadata, - &test_workload_identity(), - ) - .unwrap_err(); - assert!( - err.message() - .contains("reserved for the OpenShell workspace") - ); - - let ancestor_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ +fn container_creation_allows_mounts_covering_working_dir() { + let mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ "mounts": [{"type": "tmpfs", "target": "/workspace"}] })) .unwrap(); - let nested_metadata = DockerImageMetadata { - working_dir: "/workspace/project".to_string(), - volumes: Vec::new(), - ..metadata.clone() - }; - let err = build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &ancestor_mount, - None, - &nested_metadata, - &test_workload_identity(), - ) - .unwrap_err(); - assert!( - err.message() - .contains("reserved for the OpenShell workspace") - ); - - let nested_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ - "mounts": [{"type": "tmpfs", "target": "/workspace/cache"}] - })) - .unwrap(); - build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &nested_mount, - None, - &metadata, - &test_workload_identity(), - ) - .expect("nested workspace mounts remain supported"); - - let compatibility_path_mount: DockerSandboxDriverConfig = - serde_json::from_value(serde_json::json!({ - "mounts": [{"type": "tmpfs", "target": "/sandbox"}] - })) - .unwrap(); - build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &compatibility_path_mount, - None, - &metadata, - &test_workload_identity(), - ) - .expect("/sandbox remains mountable when the inspected workspace is elsewhere"); + for workdir in ["/workspace", "/workspace/project"] { + let image = DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "1234:1235".to_string(), + working_dir: workdir.to_string(), + volumes: Vec::new(), + }; + build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &mount, + None, + &image, + &test_workload_identity(), + ) + .expect("driver mounts may cover the workload workspace"); + } } #[test] diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index d621001527..78a6891a9d 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -94,13 +94,12 @@ environment belong to agent children, never the supervisor process. OpenShell reads `WORKDIR` from the workload image. If it is unset, `/`, or `/sandbox`, OpenShell uses its managed `/sandbox` workspace. A custom path must -be absolute, with no `.` or `..` segments. It cannot overlap container system -paths (`/proc`, `/sys`, `/dev`) or mounts still used in the workload: -`/.openshell` for the control channel, `/opt/openshell/bin` for the sandbox -runtime, and `/run/openshell-supervisor-ca` for generated CA material. Paths -used only by the separate supervisor are allowed. Image and driver mounts may -be inside the workspace, but cannot replace the workspace path or one of its -parents. +be absolute, with no `.` or `..` segments. It cannot overlap mounts still +used in the workload: `/.openshell` for the control channel, +`/opt/openshell/bin` for the sandbox runtime, and +`/run/openshell-supervisor-ca` for generated CA material. Supervisor-only paths +outside those private mounts are allowed. Image and driver mounts may overlap +the workspace; a conflicting mount can prevent the workload from starting. For a custom path, Podman mounts a persistent workspace volume there. When the volume is first created, Podman copies any files already in that image directory @@ -139,8 +138,8 @@ User `bind`, `volume`, `tmpfs`, and `image` mounts and CDI GPU selection remain native Podman features and apply only to the workload. Bind mounts require the operator's `enable_bind_mounts` opt-in and disabled label admission. Supplemental image mounts also require disabled admission. Driver JSON requires -`allow_driver_config = true`. The workload's private mounts and workspace -root cannot be replaced. User-owned volumes are never created or deleted. +`allow_driver_config = true`. The workload's private mounts cannot be +replaced. User-owned volumes are never created or deleted. See [gateway configuration](../../docs/how-it-works/gateways/configuration.mdx) for operator settings and [NETWORKING.md](NETWORKING.md) for supervisor networking. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index f09311ecd6..ce251c1dcd 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -234,10 +234,9 @@ pub struct ResolvedPodmanImage { impl ResolvedPodmanImage { /// Resolve the image metadata and reject: - /// - a malformed or relative working directory, or one overlapping runtime - /// or `OpenShell` control paths; - /// - an image volume with an invalid or reserved target; and - /// - an image volume covering the workspace or any of its ancestors. + /// - a malformed or relative working directory, or one overlapping the + /// workload's trusted runtime, control channel, or CA mount; + /// - an image volume with an invalid or reserved target. pub fn from_inspect(inspected: &ImageInspect) -> Result { let image_config = inspected.config.as_ref(); let workspace_root = driver_mounts::resolve_oci_workspace_root_for_workload( @@ -256,13 +255,6 @@ impl ResolvedPodmanImage { "invalid image-declared volume '{volume}': {error}" )) })?; - driver_mounts::validate_workspace_mount_target(volume, &workspace_root).map_err( - |_| { - ComputeDriverError::Precondition(format!( - "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" - )) - }, - )?; } } @@ -841,7 +833,6 @@ pub fn podman_driver_image_mount_sources( fn podman_user_mounts( sandbox: &DriverSandbox, enable_bind_mounts: bool, - workspace_root: &str, ) -> Result { let template = sandbox .spec @@ -853,13 +844,6 @@ fn podman_user_mounts( let config = podman_driver_config(template, enable_bind_mounts)?; let mut result = PodmanUserMounts::default(); for mount in config.mounts { - let target = match &mount { - PodmanDriverMountConfig::Bind { target, .. } - | PodmanDriverMountConfig::Volume { target, .. } - | PodmanDriverMountConfig::Tmpfs { target, .. } - | PodmanDriverMountConfig::Image { target, .. } => target, - }; - driver_mounts::validate_workspace_mount_target(target, workspace_root)?; match mount { PodmanDriverMountConfig::Bind { source, @@ -1213,7 +1197,7 @@ fn build_base_spec( .unwrap_or_default(), ); let resource_limits = build_resource_limits(sandbox, config); - let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts, &image.workspace_root) + let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) .map_err(ComputeDriverError::InvalidArgument)?; if sandbox .spec @@ -2105,7 +2089,7 @@ mod tests { } #[test] - fn resolved_image_rejects_masking_oci_volumes_but_allows_nested_volumes() { + fn resolved_image_allows_volumes_covering_or_nested_in_working_dir() { let inspect = |volume: &str| ImageInspect { id: "sha256:image".into(), config: Some(ImageConfig { @@ -2118,7 +2102,8 @@ mod tests { }), }; - assert!(ResolvedPodmanImage::from_inspect(&inspect("/workspace")).is_err()); + ResolvedPodmanImage::from_inspect(&inspect("/workspace")) + .expect("image volumes may cover a workload-owned workspace"); let image = ResolvedPodmanImage::from_inspect(&inspect("/workspace/project/cache")) .expect("image volumes nested below the workspace remain valid"); assert_eq!(image.workspace_root, "/workspace/project"); @@ -2142,6 +2127,12 @@ mod tests { .workspace_root, "/etc/openshell/tls/client" ); + assert_eq!( + ResolvedPodmanImage::from_inspect(&inspect("/proc")) + .unwrap() + .workspace_root, + "/proc" + ); } fn json_struct(value: Value) -> prost_types::Struct { @@ -3235,7 +3226,7 @@ mod tests { } #[test] - fn resolved_workspace_rejects_masking_driver_mount() { + fn resolved_workspace_allows_covering_driver_mount() { use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; let image = resolved_image("sha256:immutable", "1000:1000", "/workspace/project"); @@ -3249,7 +3240,7 @@ mod tests { }), ..Default::default() }); - let error = build_container_spec_for_image( + build_container_spec_for_image( &sandbox, &test_config(), None, @@ -3259,12 +3250,7 @@ mod tests { None, None, ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("reserved for the OpenShell workspace") - ); + .expect("driver mounts may cover a workload-owned workspace"); } #[test] diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 08053b50c5..7b547e60f5 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -125,7 +125,9 @@ enabled = false -Mount targets cannot replace the workspace root, the container root, or OpenShell paths under `/opt/openshell`, `/etc/openshell`, and `/run/openshell`. +Mount targets may overlap the workspace, but cannot cover the workload's +`/.openshell` runtime and control channel or its +`/run/openshell-supervisor-ca` mount. ## Podman Driver @@ -270,12 +272,12 @@ Set `process.run_as_user` and `process.run_as_group` in the sandbox policy to ch On Docker and Podman, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR`, `WORKDIR /`, or `WORKDIR /sandbox` use OpenShell's managed `/sandbox` workspace. Custom paths must be absolute, with no `.` or `..` -segments, and cannot overlap container system paths (`/proc`, `/sys`, `/dev`) -or private mounts still used inside the workload. Paths used only by the -separate supervisor are allowed. Image and driver mounts cannot replace the -workspace or one of its parents. +segments, and cannot overlap private mounts still used inside the workload. +Supervisor-only paths outside that private tree are allowed. Image and driver +mounts may overlap the workspace, but conflicting mounts can prevent the +workload from starting. -Docker validates the directory in the image filesystem. Podman mounts the +Docker validates the directory visible after container mounts. Podman mounts the persistent workspace volume at that path. When the volume is first created, Podman copies existing files from the image directory into it. OpenShell keeps their ownership and permissions, and fails unless the final non-root user can diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 117908e13c..5f0a9d4b01 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -280,8 +280,8 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default Docker workdir or a copied-up custom Podman workspace. -- Docker and Podman also reject an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the path before validation. Move the `VOLUME` below the workspace or remove the declaration. -- A workdir rejected as a special filesystem or workload-mount collision cannot be made valid with permissions. Move it away from `/proc`, `/sys`, `/dev`, and the sandbox runtime or control-channel paths named in the error. Paths used only by the separate supervisor are allowed. +- Docker and Podman allow image volumes and driver mounts to overlap the workdir. If the container runtime rejects conflicting mounts or the final directory is unusable, move the mount or choose another workdir. +- A workdir rejected for overlapping a private workload mount cannot be made valid with permissions. Move it away from the sandbox runtime, control-channel, or CA paths named in the error. Supervisor-only paths outside those private mounts are allowed. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. From e7738cbe0221c70e55f8e93ef5887f00dd02a76f Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 17:49:22 -0700 Subject: [PATCH 06/20] refactor(podman): keep OCI workdir support scoped to Podman Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 13 +- crates/openshell-core/src/container_paths.rs | 29 ++++ crates/openshell-core/src/driver_mounts.rs | 124 +++++++++++++++- crates/openshell-driver-docker/README.md | 15 +- crates/openshell-driver-docker/src/lib.rs | 55 +++---- crates/openshell-driver-docker/src/tests.rs | 143 +++++++++++-------- crates/openshell-sandbox/src/process.rs | 7 +- docs/how-it-works/sandboxes/runtimes.mdx | 27 ++-- e2e/rust/Cargo.toml | 2 +- e2e/rust/tests/custom_image.rs | 2 +- e2e/rust/tests/driver_config_volume.rs | 14 +- skills/debug-openshell-cluster/SKILL.md | 7 +- skills/openshell-cli/SKILL.md | 9 +- 13 files changed, 304 insertions(+), 143 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index c15ea662bc..4fb4436207 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -426,13 +426,14 @@ traffic. No untrusted child performs an identity transition. Identity-changing policy updates require sandbox recreation, while other policy updates remain live. -Docker and Podman resolve OCI `Config.User` and `Config.WorkingDir` from one -immutable image inspection. Empty, `/`, and explicit `/sandbox` values use the -managed `/sandbox` workspace. Custom paths must be normalized absolute paths -that do not overlap private mounts still inside the workload container. -Supervisor-only paths outside that private tree are not reserved. Image and +Docker retains its existing working-directory and mount validation. Podman +resolves OCI `Config.User` and `Config.WorkingDir` from one immutable image +inspection. Empty, `/`, and explicit `/sandbox` values use the managed +`/sandbox` workspace. Custom paths must be normalized absolute paths that do +not overlap private mounts inside the Podman workload. Other paths are not +reserved merely because the separate supervisor uses them. Podman image and driver mounts may cover the workspace; conflicting mounts can make the workload -unusable. Private workload mount targets remain reserved. +unusable. Podman mounts its persistent workspace volume at a custom root. When the volume is first created, Podman copies existing image-directory contents into it. diff --git a/crates/openshell-core/src/container_paths.rs b/crates/openshell-core/src/container_paths.rs index 1dab75bd0a..63511c13ff 100644 --- a/crates/openshell-core/src/container_paths.rs +++ b/crates/openshell-core/src/container_paths.rs @@ -16,6 +16,15 @@ pub const SIDECAR_RUN_ROOT: &str = "/run/openshell-sidecar"; pub const NETNS_MOUNT_ROOT: &str = "/run/netns"; pub const NETNS_IPROUTE2_ROOT: &str = "/var/run/netns"; +/// Standard Linux container namespaces that an image-selected workspace must +/// not contain or enter. +/// +/// These roots cover the default filesystems and devices defined by the OCI +/// Runtime Specification: procfs, sysfs, cgroups, device nodes, devpts, shared +/// memory, and POSIX message queues. +/// +pub const OCI_RUNTIME_MOUNT_ROOTS: &[&str] = &["/proc", "/sys", "/dev"]; + /// High-level namespaces mounted or created by `OpenShell` inside sandboxes. /// /// This is intentionally not a general Linux system-path denylist. Kernel and @@ -135,4 +144,24 @@ mod tests { ); } } + + #[test] + fn runtime_roots_cover_standard_oci_mount_destinations() { + for path in [ + "/proc", + "/dev", + "/dev/pts", + "/dev/shm", + "/dev/mqueue", + "/sys", + "/sys/fs/cgroup", + ] { + assert!( + OCI_RUNTIME_MOUNT_ROOTS + .iter() + .any(|root| Path::new(path).starts_with(root)), + "OCI runtime mount {path} is outside the reserved roots" + ); + } + } } diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index 7402f778b1..a01869bd32 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -5,7 +5,7 @@ use std::path::Path; -use crate::container_paths::CONTROL_ROOTS; +use crate::container_paths::{CONTROL_ROOTS, OCI_RUNTIME_MOUNT_ROOTS}; /// `SELinux` relabelling mode for bind mounts. /// @@ -81,7 +81,8 @@ pub fn validate_mount_subpath(subpath: &str) -> Result<(), String> { /// Validate a container-side mount target for user-supplied driver mounts. /// -/// Drivers may apply additional checks for mounts used by their workload. +/// Workspace collisions depend on the inspected image's resolved working +/// directory and are checked separately by `validate_workspace_mount_target`. pub fn validate_container_mount_target(target: &str) -> Result<(), String> { validate_container_mount_target_for_workload(target, CONTROL_ROOTS) } @@ -113,9 +114,18 @@ pub fn validate_container_mount_target_for_workload( /// value and the path passed to the supervisor cannot be interpreted /// differently. pub fn resolve_oci_workspace_root(working_dir: &str) -> Result { - // The sandbox runtime checks syntax again; each compute driver checks - // its own workload mounts before admitting the workspace. - resolve_oci_workspace_root_for_workload(working_dir, &[]) + if working_dir.is_empty() || working_dir == "/" { + return Ok(DEFAULT_WORKSPACE_ROOT.to_string()); + } + let workspace_root = normalize_absolute_container_path(working_dir, "OCI WorkingDir")?; + for runtime_path in OCI_RUNTIME_MOUNT_ROOTS { + validate_workspace_reserved_path(&workspace_root, runtime_path, "OCI runtime mount")?; + } + for control_path in CONTROL_ROOTS { + validate_workspace_control_path(&workspace_root, control_path)?; + } + + Ok(workspace_root) } /// Resolve a workspace against paths still mounted inside this workload. @@ -192,8 +202,25 @@ fn validate_workspace_reserved_path( Ok(()) } -/// Reject a user-supplied mount that would replace or contain a driver-managed -/// workspace root. Kubernetes uses this for its fixed workspace mount. +/// Reject a mount that contains or is contained by a runtime-configured +/// `OpenShell` control path, such as the sandbox SSH socket. +pub fn validate_mount_control_path(target: &str, control_path: &str) -> Result<(), String> { + let normalized_target = normalize_absolute_container_path(target, "mount target")?; + let normalized_control = + normalize_absolute_container_path(control_path, "OpenShell control path")?; + if paths_overlap( + Path::new(&normalized_target), + Path::new(&normalized_control), + ) { + return Err(format!( + "mount target '{target}' conflicts with OpenShell control path '{control_path}'" + )); + } + Ok(()) +} + +/// Reject a user-supplied mount that would replace or contain the resolved +/// workspace root. Mounts below the workspace remain valid. pub fn validate_workspace_mount_target(target: &str, workspace_root: &str) -> Result<(), String> { let normalized_target = normalize_mount_target(target); if path_is_or_under(Path::new(workspace_root), Path::new(&normalized_target)) { @@ -274,6 +301,89 @@ mod tests { } } + #[test] + fn oci_workspace_root_rejects_runtime_and_openshell_control_path_collisions() { + for invalid in [ + "/proc", + "/proc/self", + "/sys", + "/sys/fs/cgroup", + "/dev", + "/dev/shm", + "/etc", + "/opt", + "/opt/openshell", + "/opt/openshell/bin/project", + "/etc/openshell/tls/client", + "/etc/openshell/auth", + "/etc/openshell/skills", + "/etc/openshell-tls", + "/run", + "/run/openshell/cache", + "/run/openshell-sidecar/control.sock", + "/run/netns/project", + "/var/run/netns/project", + ] { + assert!( + resolve_oci_workspace_root(invalid).is_err(), + "expected control-path workspace '{invalid}' to be rejected" + ); + } + + for valid in [ + "/app", + "/etc/project", + "/home/app", + "/opt/app", + "/usr/bin/project", + "/usr/src/app", + "/var/lib/app", + "/var/app/current", + "/var/task", + "/var/www/app", + "/processor", + "/system", + "/device", + ] { + assert_eq!( + resolve_oci_workspace_root(valid).unwrap(), + valid, + "expected application workspace '{valid}' to remain valid" + ); + } + } + + #[test] + fn container_target_rejects_reserved_openshell_tls_legacy_path() { + let err = validate_container_mount_target("/etc/openshell-tls/proxy/client").unwrap_err(); + + assert!(err.contains("/etc/openshell-tls")); + } + + #[test] + fn container_target_rejects_reserved_openshell_tree() { + let err = validate_container_mount_target("/etc/openshell/tls/client").unwrap_err(); + + assert!(err.contains("/etc/openshell")); + } + + #[test] + fn container_target_does_not_prefix_match_unrelated_paths() { + validate_container_mount_target("/etc/openshell-tools").unwrap(); + validate_container_mount_target("/run/openshell-tools").unwrap(); + } + + #[test] + fn mount_target_rejects_runtime_configured_control_path_overlap() { + for target in ["/custom", "/custom/ssh.sock", "/custom/ssh.sock/cache"] { + assert!( + validate_mount_control_path(target, "/custom/ssh.sock").is_err(), + "expected '{target}' to conflict with the configured control path" + ); + } + validate_mount_control_path("/custom-other", "/custom/ssh.sock").unwrap(); + } + #[test] fn oci_workspace_root_only_reserves_selected_workload_paths() { let reserved = &["/control"]; diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 7dde4011d7..a25c455eea 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -66,12 +66,10 @@ already exist without symlink components. The resolved identity must be able to traverse every parent and write and enter the workdir; OpenShell does not change its ownership or mode. -The workdir cannot overlap the workload's `/.openshell` runtime/channel tree -or its `/run/openshell-supervisor-ca` mount. Image `VOLUME` declarations and -user mounts may cover the workdir; Docker's final mount layout determines -whether that directory is usable. OpenShell asks the kernel to validate access -under the final identity, so POSIX ACL and host LSM decisions remain -authoritative. +Image `VOLUME` declarations and user mounts must not cover the workdir, one of +its parents, or the reserved `/.openshell` runtime/channel tree. OpenShell asks +the kernel to validate access under the final identity, so POSIX ACL and host +LSM decisions remain authoritative. ## Container Contract @@ -115,9 +113,8 @@ mount types are: Host bind mounts are disabled by default because they expose daemon-host paths to sandbox requests. User bind and volume mounts are read-only by default. -Targets must be absolute, normalized paths and cannot overlap private mounts -still used inside the workload. They may overlap the workspace; paths outside -the workload's private tree that are used only by the supervisor are allowed. +Targets must be absolute, normalized paths and cannot overlap the workspace +root or OpenShell control paths. Example: diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index ed3245b743..1f4c91f5e4 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -105,10 +105,6 @@ const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/.openshell/channel/sandbox/bootstrap. const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/sandbox/control.sock"; const BOUNDARY_CERTIFICATE_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.crt"; const BOUNDARY_PRIVATE_KEY_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.key"; -const WORKLOAD_RESERVED_PATHS: &[&str] = &[ - "/.openshell", - openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, -]; const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/supervisor"; const SUPERVISOR_PROXY_AUTH_MOUNT_PATH: &str = "/.openshell/supervisor/upstream-proxy-auth"; const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = @@ -3763,8 +3759,7 @@ fn docker_bind_string( "bind source path does not exist: {source}" ))); } - driver_mounts::validate_container_mount_target_for_workload(target, WORKLOAD_RESERVED_PATHS) - .map_err(Status::failed_precondition)?; + driver_mounts::validate_container_mount_target(target).map_err(Status::failed_precondition)?; let normalized_target = driver_mounts::normalize_mount_target(target); let mut opts = Vec::new(); @@ -3902,11 +3897,8 @@ fn validate_docker_driver_mounts( )); } }; - driver_mounts::validate_container_mount_target_for_workload( - target, - WORKLOAD_RESERVED_PATHS, - ) - .map_err(Status::failed_precondition)?; + driver_mounts::validate_container_mount_target(target) + .map_err(Status::failed_precondition)?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(Status::failed_precondition(format!( @@ -4532,11 +4524,8 @@ async fn prepare_docker_boundary_files( gpu_requested: bool, ) -> Result<(), Status> { let directory = docker_boundary_state_dir(sandbox, config)?; - let workspace_root = driver_mounts::resolve_oci_workspace_root_for_workload( - &image.working_dir, - WORKLOAD_RESERVED_PATHS, - ) - .map_err(Status::failed_precondition)?; + let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) + .map_err(Status::failed_precondition)?; let launch_authentication = sandbox .spec .as_ref() @@ -5663,21 +5652,35 @@ fn build_container_create_body_for_image( .as_ref() .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; let resource_limits = docker_resource_limits(template)?; - driver_mounts::resolve_oci_workspace_root_for_workload( - &image.working_dir, - WORKLOAD_RESERVED_PATHS, - ) - .map_err(Status::failed_precondition)?; + let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) + .map_err(Status::failed_precondition)?; + driver_mounts::validate_workspace_control_path(&workspace_root, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; for volume in &image.volumes { - driver_mounts::validate_container_mount_target_for_workload( - volume, - WORKLOAD_RESERVED_PATHS, - ) - .map_err(|error| { + driver_mounts::validate_container_mount_target(volume).map_err(|error| { Status::failed_precondition(format!( "invalid image-declared volume '{volume}': {error}" )) })?; + driver_mounts::validate_workspace_mount_target(volume, &workspace_root).map_err(|_| { + Status::failed_precondition(format!( + "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" + )) + })?; + driver_mounts::validate_mount_control_path(volume, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; + } + for mount in &driver_config.mounts { + let target = match mount { + DockerDriverMountConfig::Bind { target, .. } + | DockerDriverMountConfig::Volume { target, .. } + | DockerDriverMountConfig::Tmpfs { target, .. } + | DockerDriverMountConfig::Image { target, .. } => target, + }; + driver_mounts::validate_workspace_mount_target(target, &workspace_root) + .map_err(Status::failed_precondition)?; + driver_mounts::validate_mount_control_path(target, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; } let mut user_mounts = docker_driver_mounts(driver_config)?; user_mounts.push(Mount { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 82443d09ba..c5036f4547 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -1551,10 +1551,10 @@ fn container_creation_rejects_invalid_oci_working_dir() { #[test] fn container_creation_rejects_openshell_control_path_working_dir() { - let mut metadata = DockerImageMetadata { + let metadata = DockerImageMetadata { id: "sha256:immutable".to_string(), user: "1234:1235".to_string(), - working_dir: "/.openshell/runtime/project".to_string(), + working_dir: "/opt/openshell/bin/project".to_string(), volumes: Vec::new(), }; let err = build_container_create_body_for_image( @@ -1569,74 +1569,112 @@ fn container_creation_rejects_openshell_control_path_working_dir() { assert_eq!(err.code(), tonic::Code::FailedPrecondition); assert!(err.message().contains("OpenShell control path")); +} - metadata.working_dir = "/opt/openshell/bin/project".to_string(); - build_container_create_body_for_image( - &test_sandbox(), +#[test] +fn container_creation_rejects_image_volume_that_masks_working_dir() { + let sandbox = test_sandbox(); + let metadata = DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "1234:1235".to_string(), + working_dir: "/workspace/project".to_string(), + volumes: vec!["/workspace".to_string()], + }; + + let error = build_container_create_body_for_image( + &sandbox, &runtime_config(), &DockerSandboxDriverConfig::default(), None, &metadata, &test_workload_identity(), ) - .expect("supervisor-only paths are not reserved in the workload"); + .unwrap_err(); - metadata.working_dir = "/proc".to_string(); - build_container_create_body_for_image( + assert!( + error + .message() + .contains("masks OCI WorkingDir '/workspace/project'") + ); +} + +#[test] +fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts() { + let metadata = DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "1234:1235".to_string(), + working_dir: "/workspace".to_string(), + volumes: Vec::new(), + }; + let root_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": "/workspace"}] + })) + .unwrap(); + let err = build_container_create_body_for_image( &test_sandbox(), &runtime_config(), - &DockerSandboxDriverConfig::default(), + &root_mount, None, &metadata, &test_workload_identity(), ) - .expect("OCI system paths are not rejected before workload launch"); -} + .unwrap_err(); + assert!( + err.message() + .contains("reserved for the OpenShell workspace") + ); -#[test] -fn container_creation_allows_image_volume_covering_working_dir() { - let sandbox = test_sandbox(); - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), + let ancestor_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": "/workspace"}] + })) + .unwrap(); + let nested_metadata = DockerImageMetadata { working_dir: "/workspace/project".to_string(), - volumes: vec!["/workspace".to_string()], + volumes: Vec::new(), + ..metadata.clone() }; + let err = build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &ancestor_mount, + None, + &nested_metadata, + &test_workload_identity(), + ) + .unwrap_err(); + assert!( + err.message() + .contains("reserved for the OpenShell workspace") + ); + let nested_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": "/workspace/cache"}] + })) + .unwrap(); build_container_create_body_for_image( - &sandbox, + &test_sandbox(), &runtime_config(), - &DockerSandboxDriverConfig::default(), + &nested_mount, None, &metadata, &test_workload_identity(), ) - .expect("image volumes may cover a workload-owned workspace"); -} + .expect("nested workspace mounts remain supported"); -#[test] -fn container_creation_allows_mounts_covering_working_dir() { - let mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ - "mounts": [{"type": "tmpfs", "target": "/workspace"}] - })) - .unwrap(); - for workdir in ["/workspace", "/workspace/project"] { - let image = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: workdir.to_string(), - volumes: Vec::new(), - }; - build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &mount, - None, - &image, - &test_workload_identity(), - ) - .expect("driver mounts may cover the workload workspace"); - } + let compatibility_path_mount: DockerSandboxDriverConfig = + serde_json::from_value(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": "/sandbox"}] + })) + .unwrap(); + build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &compatibility_path_mount, + None, + &metadata, + &test_workload_identity(), + ) + .expect("/sandbox remains mountable when the inspected workspace is elsewhere"); } #[test] @@ -2107,7 +2145,7 @@ fn driver_config_rejects_reserved_mount_targets() { "mounts": [{ "type": "volume", "source": "work-nfs", - "target": "/.openshell/runtime" + "target": "/etc/openshell/auth" }] }))); @@ -2115,19 +2153,6 @@ fn driver_config_rejects_reserved_mount_targets() { assert_eq!(err.code(), tonic::Code::FailedPrecondition); assert!(err.message().contains("reserved OpenShell path")); - - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{"type": "volume", "source": "work-nfs", "target": "/etc/openshell/auth"}] - }))); - build_container_create_body(&sandbox, &runtime_config()) - .expect("supervisor-only paths are not reserved in the workload"); } #[test] diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 7b61d30030..1b831d6163 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -1623,8 +1623,11 @@ fn validated_workspace_components( let root_str = root .to_str() .ok_or_else(|| miette::miette!("workspace path must be valid UTF-8"))?; - let validated_root = openshell_core::driver_mounts::resolve_oci_workspace_root(root_str) - .map_err(|error| miette::miette!(error))?; + // The driver has already checked collisions with its workload mounts. + // Recheck syntax here without imposing Docker's broader path reservations. + let validated_root = + openshell_core::driver_mounts::resolve_oci_workspace_root_for_workload(root_str, &[]) + .map_err(|error| miette::miette!(error))?; if Path::new(&validated_root) != root || (!allow_managed_fallback && validated_root == openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 7b547e60f5..c844225224 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -125,9 +125,7 @@ enabled = false -Mount targets may overlap the workspace, but cannot cover the workload's -`/.openshell` runtime and control channel or its -`/run/openshell-supervisor-ca` mount. +Mount targets cannot replace the workspace root, the container root, or OpenShell paths under `/opt/openshell`, `/etc/openshell`, and `/run/openshell`. ## Podman Driver @@ -269,17 +267,12 @@ Set `process.run_as_user` and `process.run_as_group` in the sandbox policy to ch | Kubernetes | OpenShift SCC namespace annotations, otherwise `1000`. Override with `sandbox_uid` and `sandbox_gid`. | | MicroVM | The image's `sandbox` account, otherwise `1000`. Override with `sandbox_uid` and `sandbox_gid`. | -On Docker and Podman, the image's `WORKDIR` becomes the workspace. Images with -no `WORKDIR`, `WORKDIR /`, or `WORKDIR /sandbox` use OpenShell's managed -`/sandbox` workspace. Custom paths must be absolute, with no `.` or `..` -segments, and cannot overlap private mounts still used inside the workload. -Supervisor-only paths outside that private tree are allowed. Image and driver -mounts may overlap the workspace, but conflicting mounts can prevent the -workload from starting. - -Docker validates the directory visible after container mounts. Podman mounts the -persistent workspace volume at that path. When the volume is first created, -Podman copies existing files from the image directory into it. OpenShell keeps -their ownership and permissions, and fails unless the final non-root user can -reach and write the directory. Test custom images with the Podman configuration -you will use. Kubernetes and MicroVM continue to use `/sandbox`. +On Docker, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR`, `/`, or `/sandbox` use `/sandbox`. Any other `WORKDIR` must exist in the image and be writable by the sandbox user. + +Podman also uses the image's `WORKDIR` as the workspace, falling back to +`/sandbox` for an empty, `/`, or `/sandbox` value. A custom path must be an +absolute, normalized path that does not overlap Podman's private workload +mounts. Podman mounts the persistent workspace volume there and copies existing +image-directory files into a new volume. OpenShell preserves their permissions +and requires the final non-root user to be able to write there. Kubernetes and +MicroVM continue to use `/sandbox`. diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 2c431a107f..b492c8ac86 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -56,7 +56,7 @@ required-features = ["e2e-vm"] [[test]] name = "custom_image" path = "tests/custom_image.rs" -required-features = ["e2e-local-container-driver"] +required-features = ["e2e-docker"] [[test]] name = "rootfs_tar" diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs index 1618290f1d..96fdd56612 100644 --- a/e2e/rust/tests/custom_image.rs +++ b/e2e/rust/tests/custom_image.rs @@ -243,7 +243,7 @@ async fn sandbox_rejects_image_workdir_that_would_require_new_authority() { } Err(error) => error, }; - let message = error; + let message = error.to_string(); assert!( (message.contains("WorkspaceValidationFailed") && message.contains("WorkingDir")) || message.contains("subsystem request failed") diff --git a/e2e/rust/tests/driver_config_volume.rs b/e2e/rust/tests/driver_config_volume.rs index 9d35bd00d6..bcef6a429c 100644 --- a/e2e/rust/tests/driver_config_volume.rs +++ b/e2e/rust/tests/driver_config_volume.rs @@ -17,7 +17,7 @@ use bollard::query_parameters::{ RemoveVolumeOptionsBuilder, StartContainerOptions, WaitContainerOptions, }; use futures_util::TryStreamExt; -#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] +#[cfg(feature = "e2e-docker")] use openshell_e2e::harness::container::ImageGuard; use openshell_e2e::harness::container::e2e_driver; use openshell_e2e::harness::sandbox::SandboxGuard; @@ -26,9 +26,9 @@ use serde_json::{Map, Value}; const TEST_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; const VOLUME_TARGET: &str = "/sandbox/e2e-volume"; const BIND_TARGET: &str = "/sandbox/e2e-bind"; -#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] +#[cfg(feature = "e2e-docker")] const OCI_VOLUME_TARGET: &str = "/workspace/project/e2e-volume"; -#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] +#[cfg(feature = "e2e-docker")] const OCI_USER_DOCKERFILE: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ @@ -119,12 +119,12 @@ async fn sandbox_mounts_existing_driver_config_volume() { } #[tokio::test] -#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] +#[cfg(feature = "e2e-docker")] async fn oci_workspace_preparation_skips_nested_volume_ownership() { let driver = e2e_driver().expect("OPENSHELL_E2E_DRIVER must be set by the e2e wrapper"); assert!( - matches!(driver.as_str(), "docker" | "podman"), - "OCI workspace mount e2e requires docker or podman, got {driver}" + driver == "docker", + "OCI workspace mount e2e requires docker, got {driver}" ); let volume = VolumeGuard::create(&driver) @@ -305,7 +305,7 @@ async fn verify_volume(volume: &VolumeGuard) -> Result<(), String> { Ok(()) } -#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] +#[cfg(feature = "e2e-docker")] async fn verify_volume_ownership(volume: &VolumeGuard) -> Result<(), String> { let output = run_volume_container( volume, diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 5f0a9d4b01..f89c5f1a7b 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -279,9 +279,10 @@ Common findings: - Gateway process stopped: inspect exit status and logs. - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. -- Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default Docker workdir or a copied-up custom Podman workspace. -- Docker and Podman allow image volumes and driver mounts to overlap the workdir. If the container runtime rejects conflicting mounts or the final directory is unusable, move the mount or choose another workdir. -- A workdir rejected for overlapping a private workload mount cannot be made valid with permissions. Move it away from the sandbox runtime, control-channel, or CA paths named in the error. Supervisor-only paths outside those private mounts are allowed. +- Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default image workdir. +- Docker also rejects an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. +- A Docker workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. +- For Podman, a custom workdir must not overlap the workload's private mounts. The volume keeps files copied from the image without changing their permissions; ensure the sandbox user can write the resulting directory. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index ce116c4797..c77a6e487c 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -678,11 +678,10 @@ Explicit numeric fields may use any UID/GID from `1` through Warn users that low IDs can inherit permissions from matching accounts, image files, mounted volumes, or devices. -Docker and Podman gateways also use a normalized absolute OCI `WORKDIR` as the -workspace. Empty, `/`, and explicit `/sandbox` declarations use the managed -`/sandbox` fallback. Podman preserves normal volume copy-up and does not repair -custom workspace ownership or mode; the final identity must be able to traverse -and write the directory. +Podman gateways use a normalized absolute OCI `WORKDIR` as the workspace. +Empty, `/`, and explicit `/sandbox` declarations use the managed `/sandbox` +fallback. For a custom path, the final identity must be able to traverse and +write the directory in the persistent volume. ### Forward ports From 917ce4180525c8656361824da0fead0a67a19900 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 18:05:04 -0700 Subject: [PATCH 07/20] refactor(podman): reuse shared workspace path validation Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 45 +++---- crates/openshell-core/src/driver_mounts.rs | 55 +------- crates/openshell-driver-podman/README.md | 10 +- .../openshell-driver-podman/src/container.rs | 124 ++++++++++-------- crates/openshell-sandbox/src/process.rs | 7 +- docs/how-it-works/sandboxes/runtimes.mdx | 11 +- skills/debug-openshell-cluster/SKILL.md | 4 +- 7 files changed, 106 insertions(+), 150 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 4fb4436207..95736400bc 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -416,32 +416,25 @@ and supplementary-group set before creating the immutable workload: namespace ranges. - VM uses the configured numeric guest identity. -UID/GID zero and `u32::MAX` are invalid. Before any untrusted instruction runs, -the sandbox runtime and every child use the resolved identity with zero -capability masks. The managed Podman `/sandbox` fallback may start its trusted -runtime bootstrap as container root with narrowly scoped identity and ownership -capabilities; it prepares the driver-owned workspace and drops irreversibly to -the resolved identity before reading bootstrap material or accepting control -traffic. No untrusted child performs an identity transition. Identity-changing -policy updates require sandbox recreation, while other policy updates remain -live. - -Docker retains its existing working-directory and mount validation. Podman -resolves OCI `Config.User` and `Config.WorkingDir` from one immutable image -inspection. Empty, `/`, and explicit `/sandbox` values use the managed -`/sandbox` workspace. Custom paths must be normalized absolute paths that do -not overlap private mounts inside the Podman workload. Other paths are not -reserved merely because the separate supervisor uses them. Podman image and -driver mounts may cover the workspace; conflicting mounts can make the workload -unusable. - -Podman mounts its persistent workspace volume at a custom root. When the volume -is first created, Podman copies existing image-directory contents into it. -OpenShell preserves their ownership and mode and starts directly as the final -non-root identity. An unusable path fails closed. Only the managed `/sandbox` -fallback uses the root/chown bootstrap. The separate supervisor receives the -path but does not mount the workspace. Kubernetes and VM continue to use -`/sandbox`. +UID/GID zero and `u32::MAX` are invalid. Agent commands run as the resolved +non-root user. For Podman's managed `/sandbox` workspace, trusted setup briefly +starts as root to prepare the workspace, then switches to that user before +reading bootstrap material or accepting commands. Identity-changing policy +updates require sandbox recreation, while other policy updates remain live. + +Docker uses an absolute OCI working directory as the workspace. Empty, root, +and explicit `/sandbox` values select `/sandbox`; other paths must already +exist without symlink or reserved-mount collisions and must be usable by the +resolved identity. + +Podman reads the image user and working directory from one pinned image. Empty, +`/`, and explicit `/sandbox` values use the managed `/sandbox` workspace. A +custom path must be absolute, normalized, and outside system and OpenShell +reserved paths. Image and driver mounts cannot cover the workspace. Podman +mounts the persistent workspace volume there and copies any existing image +files into it on first use. OpenShell preserves their ownership and permissions; +the final non-root user must be able to write the resulting workspace. +Kubernetes and VM use `/sandbox`. ### Executable Identity Binding diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index a01869bd32..b1a3049882 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -84,17 +84,9 @@ pub fn validate_mount_subpath(subpath: &str) -> Result<(), String> { /// Workspace collisions depend on the inspected image's resolved working /// directory and are checked separately by `validate_workspace_mount_target`. pub fn validate_container_mount_target(target: &str) -> Result<(), String> { - validate_container_mount_target_for_workload(target, CONTROL_ROOTS) -} - -/// Validate a mount target against paths used by this specific workload. -pub fn validate_container_mount_target_for_workload( - target: &str, - workload_reserved_paths: &[&str], -) -> Result<(), String> { let normalized = normalize_absolute_container_path(target, "mount target")?; let path = Path::new(&normalized); - for reserved in workload_reserved_paths { + for reserved in CONTROL_ROOTS { let reserved = Path::new(reserved); if paths_overlap(path, reserved) { return Err(format!( @@ -128,22 +120,6 @@ pub fn resolve_oci_workspace_root(working_dir: &str) -> Result { Ok(workspace_root) } -/// Resolve a workspace against paths still mounted inside this workload. -pub fn resolve_oci_workspace_root_for_workload( - working_dir: &str, - workload_reserved_paths: &[&str], -) -> Result { - if working_dir.is_empty() || working_dir == "/" { - return Ok(DEFAULT_WORKSPACE_ROOT.to_string()); - } - let workspace_root = normalize_absolute_container_path(working_dir, "OCI WorkingDir")?; - for control_path in workload_reserved_paths { - validate_workspace_control_path(&workspace_root, control_path)?; - } - - Ok(workspace_root) -} - fn normalize_absolute_container_path(value: &str, field: &str) -> Result { if value.is_empty() { return Err(format!("{field} must not be empty")); @@ -384,35 +360,6 @@ mod tests { validate_mount_control_path("/custom-other", "/custom/ssh.sock").unwrap(); } - #[test] - fn oci_workspace_root_only_reserves_selected_workload_paths() { - let reserved = &["/control"]; - for invalid in ["/control", "/control/data"] { - assert!( - resolve_oci_workspace_root_for_workload(invalid, reserved).is_err(), - "expected workspace '{invalid}' to be rejected" - ); - } - assert_eq!( - resolve_oci_workspace_root_for_workload("/etc/openshell", reserved).unwrap(), - "/etc/openshell" - ); - for path in ["/proc", "/sys", "/dev/shm"] { - assert_eq!( - resolve_oci_workspace_root_for_workload(path, reserved).unwrap(), - path - ); - } - } - - #[test] - fn container_target_uses_selected_workload_paths() { - let reserved = &["/control"]; - assert!(validate_container_mount_target_for_workload("/control/data", reserved).is_err()); - validate_container_mount_target_for_workload("/control-tools", reserved).unwrap(); - validate_container_mount_target_for_workload("/etc/openshell", reserved).unwrap(); - } - #[test] fn workspace_rejects_malformed_runtime_control_paths() { for control_path in [ diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 78a6891a9d..48f50fa88e 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -94,12 +94,10 @@ environment belong to agent children, never the supervisor process. OpenShell reads `WORKDIR` from the workload image. If it is unset, `/`, or `/sandbox`, OpenShell uses its managed `/sandbox` workspace. A custom path must -be absolute, with no `.` or `..` segments. It cannot overlap mounts still -used in the workload: `/.openshell` for the control channel, -`/opt/openshell/bin` for the sandbox runtime, and -`/run/openshell-supervisor-ca` for generated CA material. Supervisor-only paths -outside those private mounts are allowed. Image and driver mounts may overlap -the workspace; a conflicting mount can prevent the workload from starting. +be absolute, with no `.` or `..` segments. It cannot overlap `/proc`, `/sys`, +`/dev`, OpenShell-reserved paths, or the workload's private control and CA +mounts. Image volumes and driver mounts cannot cover the workspace or one of +its parents; mounts nested below it remain valid. For a custom path, Podman mounts a persistent workspace volume there. When the volume is first created, Podman copies any files already in that image directory diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index ce251c1dcd..39b475c8cf 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -75,11 +75,6 @@ const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = const SUPERVISOR_MOUNT_DIR: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_DIR; /// Full path to the supervisor binary inside sandbox containers. const SUPERVISOR_BINARY_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; -const WORKLOAD_RESERVED_PATHS: &[&str] = &[ - "/.openshell", - SUPERVISOR_MOUNT_DIR, - openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, -]; #[derive(Debug, Clone, Default, serde::Deserialize)] #[serde(default, deny_unknown_fields)] @@ -234,27 +229,36 @@ pub struct ResolvedPodmanImage { impl ResolvedPodmanImage { /// Resolve the image metadata and reject: - /// - a malformed or relative working directory, or one overlapping the - /// workload's trusted runtime, control channel, or CA mount; - /// - an image volume with an invalid or reserved target. + /// - a malformed or reserved working directory; + /// - an image volume with an invalid or reserved target, or one covering + /// the resolved workspace. pub fn from_inspect(inspected: &ImageInspect) -> Result { let image_config = inspected.config.as_ref(); - let workspace_root = driver_mounts::resolve_oci_workspace_root_for_workload( + let workspace_root = driver_mounts::resolve_oci_workspace_root( image_config.map_or("", |config| config.working_dir.as_str()), - WORKLOAD_RESERVED_PATHS, ) .map_err(ComputeDriverError::Precondition)?; + for control_path in [ + "/.openshell", + openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, + ] { + driver_mounts::validate_workspace_control_path(&workspace_root, control_path) + .map_err(ComputeDriverError::Precondition)?; + } if let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) { for volume in volumes.keys() { - driver_mounts::validate_container_mount_target_for_workload( - volume, - WORKLOAD_RESERVED_PATHS, - ) - .map_err(|error| { + validate_podman_mount_target(volume).map_err(|error| { ComputeDriverError::Precondition(format!( "invalid image-declared volume '{volume}': {error}" )) })?; + driver_mounts::validate_workspace_mount_target(volume, &workspace_root).map_err( + |_| { + ComputeDriverError::Precondition(format!( + "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" + )) + }, + )?; } } @@ -830,6 +834,17 @@ pub fn podman_driver_image_mount_sources( .collect()) } +fn validate_podman_mount_target(target: &str) -> Result<(), String> { + driver_mounts::validate_container_mount_target(target)?; + for control_path in [ + "/.openshell", + openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, + ] { + driver_mounts::validate_mount_control_path(target, control_path)?; + } + Ok(()) +} + fn podman_user_mounts( sandbox: &DriverSandbox, enable_bind_mounts: bool, @@ -861,10 +876,7 @@ fn podman_user_mounts( None => {} } driver_mounts::validate_absolute_mount_source(&source, "bind source")?; - driver_mounts::validate_container_mount_target_for_workload( - &target, - WORKLOAD_RESERVED_PATHS, - )?; + validate_podman_mount_target(&target)?; result.mounts.push(Mount { kind: "bind".into(), source, @@ -880,10 +892,7 @@ fn podman_user_mounts( } => { reject_subpath(subpath.as_deref(), "podman volume mounts")?; driver_mounts::validate_mount_source(&source, "volume source")?; - driver_mounts::validate_container_mount_target_for_workload( - &target, - WORKLOAD_RESERVED_PATHS, - )?; + validate_podman_mount_target(&target)?; result.volumes.push(NamedVolume { name: source, dest: target, @@ -909,10 +918,7 @@ fn podman_user_mounts( { options.push(format!("mode={mode:o}")); } - driver_mounts::validate_container_mount_target_for_workload( - &target, - WORKLOAD_RESERVED_PATHS, - )?; + validate_podman_mount_target(&target)?; result.mounts.push(Mount { kind: "tmpfs".into(), source: "tmpfs".into(), @@ -928,10 +934,7 @@ fn podman_user_mounts( } => { reject_subpath(subpath.as_deref(), "podman image mounts")?; driver_mounts::validate_mount_source(&source, "image source")?; - driver_mounts::validate_container_mount_target_for_workload( - &target, - WORKLOAD_RESERVED_PATHS, - )?; + validate_podman_mount_target(&target)?; result.image_volumes.push(ImageVolume { source, destination: target, @@ -1006,10 +1009,7 @@ fn validate_podman_driver_mounts( target } }; - driver_mounts::validate_container_mount_target_for_workload( - target, - WORKLOAD_RESERVED_PATHS, - )?; + validate_podman_mount_target(target)?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(format!( @@ -1199,6 +1199,26 @@ fn build_base_spec( let resource_limits = build_resource_limits(sandbox, config); let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) .map_err(ComputeDriverError::InvalidArgument)?; + for target in user_mounts + .mounts + .iter() + .map(|mount| mount.destination.as_str()) + .chain( + user_mounts + .volumes + .iter() + .map(|volume| volume.dest.as_str()), + ) + .chain( + user_mounts + .image_volumes + .iter() + .map(|volume| volume.destination.as_str()), + ) + { + driver_mounts::validate_workspace_mount_target(target, &image.workspace_root) + .map_err(ComputeDriverError::Precondition)?; + } if sandbox .spec .as_ref() @@ -2089,7 +2109,7 @@ mod tests { } #[test] - fn resolved_image_allows_volumes_covering_or_nested_in_working_dir() { + fn resolved_image_rejects_volumes_covering_working_dir() { let inspect = |volume: &str| ImageInspect { id: "sha256:image".into(), config: Some(ImageConfig { @@ -2102,15 +2122,14 @@ mod tests { }), }; - ResolvedPodmanImage::from_inspect(&inspect("/workspace")) - .expect("image volumes may cover a workload-owned workspace"); + assert!(ResolvedPodmanImage::from_inspect(&inspect("/workspace")).is_err()); let image = ResolvedPodmanImage::from_inspect(&inspect("/workspace/project/cache")) .expect("image volumes nested below the workspace remain valid"); assert_eq!(image.workspace_root, "/workspace/project"); } #[test] - fn resolved_image_allows_supervisor_only_workdir_but_reserves_workload_mounts() { + fn resolved_image_reuses_reserved_workdir_validation() { let inspect = |working_dir: &str| ImageInspect { id: "sha256:image".into(), config: Some(ImageConfig { @@ -2121,17 +2140,13 @@ mod tests { assert!(ResolvedPodmanImage::from_inspect(&inspect("/opt/openshell/bin/project")).is_err()); assert!(ResolvedPodmanImage::from_inspect(&inspect("/.openshell/channel")).is_err()); + assert!(ResolvedPodmanImage::from_inspect(&inspect("/etc/openshell/tls/client")).is_err()); + assert!(ResolvedPodmanImage::from_inspect(&inspect("/proc")).is_err()); assert_eq!( - ResolvedPodmanImage::from_inspect(&inspect("/etc/openshell/tls/client")) - .unwrap() - .workspace_root, - "/etc/openshell/tls/client" - ); - assert_eq!( - ResolvedPodmanImage::from_inspect(&inspect("/proc")) + ResolvedPodmanImage::from_inspect(&inspect("/home/app")) .unwrap() .workspace_root, - "/proc" + "/home/app" ); } @@ -3226,7 +3241,7 @@ mod tests { } #[test] - fn resolved_workspace_allows_covering_driver_mount() { + fn resolved_workspace_rejects_covering_driver_mount() { use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; let image = resolved_image("sha256:immutable", "1000:1000", "/workspace/project"); @@ -3240,7 +3255,7 @@ mod tests { }), ..Default::default() }); - build_container_spec_for_image( + let error = build_container_spec_for_image( &sandbox, &test_config(), None, @@ -3250,7 +3265,12 @@ mod tests { None, None, ) - .expect("driver mounts may cover a workload-owned workspace"); + .unwrap_err(); + assert!( + error + .to_string() + .contains("reserved for the OpenShell workspace") + ); } #[test] @@ -3498,8 +3518,8 @@ mod tests { .driver_config = Some(json_struct(serde_json::json!({ "mounts": [{"type": "volume", "source": "work-nfs", "target": "/etc/openshell/tls/client"}] }))); - try_build_container_spec_with_token(&sandbox, &config, None) - .expect("supervisor-only paths are not reserved in the workload"); + let err = try_build_container_spec_with_token(&sandbox, &config, None).unwrap_err(); + assert!(err.to_string().contains("reserved OpenShell path")); } #[test] diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 1b831d6163..7b61d30030 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -1623,11 +1623,8 @@ fn validated_workspace_components( let root_str = root .to_str() .ok_or_else(|| miette::miette!("workspace path must be valid UTF-8"))?; - // The driver has already checked collisions with its workload mounts. - // Recheck syntax here without imposing Docker's broader path reservations. - let validated_root = - openshell_core::driver_mounts::resolve_oci_workspace_root_for_workload(root_str, &[]) - .map_err(|error| miette::miette!(error))?; + let validated_root = openshell_core::driver_mounts::resolve_oci_workspace_root(root_str) + .map_err(|error| miette::miette!(error))?; if Path::new(&validated_root) != root || (!allow_managed_fallback && validated_root == openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index c844225224..a4c6b12544 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -271,8 +271,9 @@ On Docker, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR` Podman also uses the image's `WORKDIR` as the workspace, falling back to `/sandbox` for an empty, `/`, or `/sandbox` value. A custom path must be an -absolute, normalized path that does not overlap Podman's private workload -mounts. Podman mounts the persistent workspace volume there and copies existing -image-directory files into a new volume. OpenShell preserves their permissions -and requires the final non-root user to be able to write there. Kubernetes and -MicroVM continue to use `/sandbox`. +absolute, normalized path outside system and OpenShell-reserved paths. Image +volumes and driver mounts cannot cover it. Podman mounts the persistent +workspace volume there and copies existing image-directory files into a new +volume. OpenShell preserves their permissions and requires the final non-root +user to be able to write there. Kubernetes and MicroVM continue to use +`/sandbox`. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index f89c5f1a7b..f18f1bb8b1 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -281,8 +281,8 @@ Common findings: - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default image workdir. - Docker also rejects an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. -- A Docker workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. -- For Podman, a custom workdir must not overlap the workload's private mounts. The volume keeps files copied from the image without changing their permissions; ensure the sandbox user can write the resulting directory. +- A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the reserved paths named in the error. +- Podman also rejects an image volume or driver mount that covers the workdir. The workspace volume keeps files copied from the image without changing their permissions; ensure the sandbox user can write the resulting directory. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. From 63dd10d3d2c8c3854d98129cc342c3afefbbf262 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 18:11:33 -0700 Subject: [PATCH 08/20] refactor(podman): ignore image-declared volumes during inspection Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 9 ++-- crates/openshell-driver-podman/README.md | 5 ++- crates/openshell-driver-podman/src/client.rs | 11 +---- .../openshell-driver-podman/src/container.rs | 44 +------------------ docs/how-it-works/sandboxes/runtimes.mdx | 12 ++--- skills/debug-openshell-cluster/SKILL.md | 2 +- 6 files changed, 18 insertions(+), 65 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 95736400bc..09897eb9f6 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -430,10 +430,11 @@ resolved identity. Podman reads the image user and working directory from one pinned image. Empty, `/`, and explicit `/sandbox` values use the managed `/sandbox` workspace. A custom path must be absolute, normalized, and outside system and OpenShell -reserved paths. Image and driver mounts cannot cover the workspace. Podman -mounts the persistent workspace volume there and copies any existing image -files into it on first use. OpenShell preserves their ownership and permissions; -the final non-root user must be able to write the resulting workspace. +reserved paths. Driver-configured mounts cannot cover the workspace. Podman +handles image-declared volumes, mounts the persistent workspace volume at the +resolved path, and copies any existing image files into it on first use. +OpenShell preserves their ownership and permissions; the final non-root user +must be able to write the resulting workspace. Kubernetes and VM use `/sandbox`. ### Executable Identity Binding diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 48f50fa88e..459efdcceb 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -96,8 +96,9 @@ OpenShell reads `WORKDIR` from the workload image. If it is unset, `/`, or `/sandbox`, OpenShell uses its managed `/sandbox` workspace. A custom path must be absolute, with no `.` or `..` segments. It cannot overlap `/proc`, `/sys`, `/dev`, OpenShell-reserved paths, or the workload's private control and CA -mounts. Image volumes and driver mounts cannot cover the workspace or one of -its parents; mounts nested below it remain valid. +mounts. Driver-configured mounts cannot cover the workspace or one of its +parents; mounts nested below it remain valid. Podman handles image-declared +volumes without OpenShell inspecting them. For a custom path, Podman mounts a persistent workspace volume there. When the volume is first created, Podman copies any files already in that image directory diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 84301f85d1..be96fc1b5c 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -187,8 +187,6 @@ pub struct ImageConfig { pub env: Vec, #[serde(default)] pub working_dir: String, - #[serde(default)] - pub volumes: Option>, } /// A container summary returned by the list API. @@ -1167,7 +1165,7 @@ mod tests { "inspect-image", vec![StubResponse::new( StatusCode::OK, - r#"{"Id":"sha256:immutable","Config":{"User":"app:staff","Env":["A=one"],"WorkingDir":"/workspace/project","Volumes":{"/workspace/project/cache":{}}}}"#, + r#"{"Id":"sha256:immutable","Config":{"User":"app:staff","Env":["A=one"],"WorkingDir":"/workspace/project"}}"#, )], ); let client = PodmanClient::new(socket_path.clone()); @@ -1189,13 +1187,6 @@ mod tests { .map(|config| config.working_dir.as_str()), Some("/workspace/project") ); - assert!( - image - .config - .as_ref() - .and_then(|config| config.volumes.as_ref()) - .is_some_and(|volumes| volumes.contains_key("/workspace/project/cache")) - ); handle.await.expect("stub task should finish"); assert_eq!( request_log diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 39b475c8cf..ead90a8cd5 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -228,10 +228,8 @@ pub struct ResolvedPodmanImage { } impl ResolvedPodmanImage { - /// Resolve the image metadata and reject: - /// - a malformed or reserved working directory; - /// - an image volume with an invalid or reserved target, or one covering - /// the resolved workspace. + /// Resolve the image metadata and reject a malformed or reserved working + /// directory. Podman owns image-declared volume handling. pub fn from_inspect(inspected: &ImageInspect) -> Result { let image_config = inspected.config.as_ref(); let workspace_root = driver_mounts::resolve_oci_workspace_root( @@ -245,23 +243,6 @@ impl ResolvedPodmanImage { driver_mounts::validate_workspace_control_path(&workspace_root, control_path) .map_err(ComputeDriverError::Precondition)?; } - if let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) { - for volume in volumes.keys() { - validate_podman_mount_target(volume).map_err(|error| { - ComputeDriverError::Precondition(format!( - "invalid image-declared volume '{volume}': {error}" - )) - })?; - driver_mounts::validate_workspace_mount_target(volume, &workspace_root).map_err( - |_| { - ComputeDriverError::Precondition(format!( - "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" - )) - }, - )?; - } - } - Ok(Self { id: inspected.id.clone(), oci_user: image_config @@ -1895,7 +1876,6 @@ mod tests { "HTTP_PROXY=http://bypass".into(), ], working_dir: working_dir.to_string(), - volumes: None, }), }) .unwrap() @@ -2108,26 +2088,6 @@ mod tests { ); } - #[test] - fn resolved_image_rejects_volumes_covering_working_dir() { - let inspect = |volume: &str| ImageInspect { - id: "sha256:image".into(), - config: Some(ImageConfig { - working_dir: "/workspace/project".into(), - volumes: Some(std::collections::HashMap::from([( - volume.into(), - Value::Null, - )])), - ..Default::default() - }), - }; - - assert!(ResolvedPodmanImage::from_inspect(&inspect("/workspace")).is_err()); - let image = ResolvedPodmanImage::from_inspect(&inspect("/workspace/project/cache")) - .expect("image volumes nested below the workspace remain valid"); - assert_eq!(image.workspace_root, "/workspace/project"); - } - #[test] fn resolved_image_reuses_reserved_workdir_validation() { let inspect = |working_dir: &str| ImageInspect { diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index a4c6b12544..a8a010fe95 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -271,9 +271,9 @@ On Docker, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR` Podman also uses the image's `WORKDIR` as the workspace, falling back to `/sandbox` for an empty, `/`, or `/sandbox` value. A custom path must be an -absolute, normalized path outside system and OpenShell-reserved paths. Image -volumes and driver mounts cannot cover it. Podman mounts the persistent -workspace volume there and copies existing image-directory files into a new -volume. OpenShell preserves their permissions and requires the final non-root -user to be able to write there. Kubernetes and MicroVM continue to use -`/sandbox`. +absolute, normalized path outside system and OpenShell-reserved paths. +Driver-configured mounts cannot cover it; Podman handles image-declared volumes. +Podman mounts the persistent workspace volume there and copies existing +image-directory files into a new volume. OpenShell preserves their permissions +and requires the final non-root user to be able to write there. Kubernetes and +MicroVM continue to use `/sandbox`. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index f18f1bb8b1..3e0f70ad75 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -282,7 +282,7 @@ Common findings: - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default image workdir. - Docker also rejects an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. - A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the reserved paths named in the error. -- Podman also rejects an image volume or driver mount that covers the workdir. The workspace volume keeps files copied from the image without changing their permissions; ensure the sandbox user can write the resulting directory. +- Podman rejects driver-configured mounts that cover the workdir but leaves image-declared volumes to Podman. The workspace volume keeps files copied from the image without changing their permissions; ensure the sandbox user can write the resulting directory. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. From aeb1080fdde69d4faccc0747c5edac1c8572fdaa Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 18:17:01 -0700 Subject: [PATCH 09/20] fix(podman): restore image volume path inspection Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 9 ++-- crates/openshell-driver-podman/README.md | 5 +-- crates/openshell-driver-podman/src/client.rs | 11 ++++- .../openshell-driver-podman/src/container.rs | 43 ++++++++++++++++++- docs/how-it-works/sandboxes/runtimes.mdx | 10 ++--- skills/debug-openshell-cluster/SKILL.md | 2 +- 6 files changed, 63 insertions(+), 17 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 09897eb9f6..95736400bc 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -430,11 +430,10 @@ resolved identity. Podman reads the image user and working directory from one pinned image. Empty, `/`, and explicit `/sandbox` values use the managed `/sandbox` workspace. A custom path must be absolute, normalized, and outside system and OpenShell -reserved paths. Driver-configured mounts cannot cover the workspace. Podman -handles image-declared volumes, mounts the persistent workspace volume at the -resolved path, and copies any existing image files into it on first use. -OpenShell preserves their ownership and permissions; the final non-root user -must be able to write the resulting workspace. +reserved paths. Image and driver mounts cannot cover the workspace. Podman +mounts the persistent workspace volume there and copies any existing image +files into it on first use. OpenShell preserves their ownership and permissions; +the final non-root user must be able to write the resulting workspace. Kubernetes and VM use `/sandbox`. ### Executable Identity Binding diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 459efdcceb..48f50fa88e 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -96,9 +96,8 @@ OpenShell reads `WORKDIR` from the workload image. If it is unset, `/`, or `/sandbox`, OpenShell uses its managed `/sandbox` workspace. A custom path must be absolute, with no `.` or `..` segments. It cannot overlap `/proc`, `/sys`, `/dev`, OpenShell-reserved paths, or the workload's private control and CA -mounts. Driver-configured mounts cannot cover the workspace or one of its -parents; mounts nested below it remain valid. Podman handles image-declared -volumes without OpenShell inspecting them. +mounts. Image volumes and driver mounts cannot cover the workspace or one of +its parents; mounts nested below it remain valid. For a custom path, Podman mounts a persistent workspace volume there. When the volume is first created, Podman copies any files already in that image directory diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index be96fc1b5c..84301f85d1 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -187,6 +187,8 @@ pub struct ImageConfig { pub env: Vec, #[serde(default)] pub working_dir: String, + #[serde(default)] + pub volumes: Option>, } /// A container summary returned by the list API. @@ -1165,7 +1167,7 @@ mod tests { "inspect-image", vec![StubResponse::new( StatusCode::OK, - r#"{"Id":"sha256:immutable","Config":{"User":"app:staff","Env":["A=one"],"WorkingDir":"/workspace/project"}}"#, + r#"{"Id":"sha256:immutable","Config":{"User":"app:staff","Env":["A=one"],"WorkingDir":"/workspace/project","Volumes":{"/workspace/project/cache":{}}}}"#, )], ); let client = PodmanClient::new(socket_path.clone()); @@ -1187,6 +1189,13 @@ mod tests { .map(|config| config.working_dir.as_str()), Some("/workspace/project") ); + assert!( + image + .config + .as_ref() + .and_then(|config| config.volumes.as_ref()) + .is_some_and(|volumes| volumes.contains_key("/workspace/project/cache")) + ); handle.await.expect("stub task should finish"); assert_eq!( request_log diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index ead90a8cd5..0ec102e422 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -228,8 +228,10 @@ pub struct ResolvedPodmanImage { } impl ResolvedPodmanImage { - /// Resolve the image metadata and reject a malformed or reserved working - /// directory. Podman owns image-declared volume handling. + /// Resolve the image metadata and reject: + /// - a malformed or reserved working directory; + /// - an image volume with an invalid or reserved target, or one covering + /// the resolved workspace. pub fn from_inspect(inspected: &ImageInspect) -> Result { let image_config = inspected.config.as_ref(); let workspace_root = driver_mounts::resolve_oci_workspace_root( @@ -243,6 +245,22 @@ impl ResolvedPodmanImage { driver_mounts::validate_workspace_control_path(&workspace_root, control_path) .map_err(ComputeDriverError::Precondition)?; } + if let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) { + for volume in volumes.keys() { + validate_podman_mount_target(volume).map_err(|error| { + ComputeDriverError::Precondition(format!( + "invalid image-declared volume '{volume}': {error}" + )) + })?; + driver_mounts::validate_workspace_mount_target(volume, &workspace_root).map_err( + |_| { + ComputeDriverError::Precondition(format!( + "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" + )) + }, + )?; + } + } Ok(Self { id: inspected.id.clone(), oci_user: image_config @@ -1876,6 +1894,7 @@ mod tests { "HTTP_PROXY=http://bypass".into(), ], working_dir: working_dir.to_string(), + volumes: None, }), }) .unwrap() @@ -2088,6 +2107,26 @@ mod tests { ); } + #[test] + fn resolved_image_rejects_volumes_covering_working_dir() { + let inspect = |volume: &str| ImageInspect { + id: "sha256:image".into(), + config: Some(ImageConfig { + working_dir: "/workspace/project".into(), + volumes: Some(std::collections::HashMap::from([( + volume.into(), + Value::Null, + )])), + ..Default::default() + }), + }; + + assert!(ResolvedPodmanImage::from_inspect(&inspect("/workspace")).is_err()); + let image = ResolvedPodmanImage::from_inspect(&inspect("/workspace/project/cache")) + .expect("image volumes nested below the workspace remain valid"); + assert_eq!(image.workspace_root, "/workspace/project"); + } + #[test] fn resolved_image_reuses_reserved_workdir_validation() { let inspect = |working_dir: &str| ImageInspect { diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index a8a010fe95..ead8b55179 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -272,8 +272,8 @@ On Docker, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR` Podman also uses the image's `WORKDIR` as the workspace, falling back to `/sandbox` for an empty, `/`, or `/sandbox` value. A custom path must be an absolute, normalized path outside system and OpenShell-reserved paths. -Driver-configured mounts cannot cover it; Podman handles image-declared volumes. -Podman mounts the persistent workspace volume there and copies existing -image-directory files into a new volume. OpenShell preserves their permissions -and requires the final non-root user to be able to write there. Kubernetes and -MicroVM continue to use `/sandbox`. +Image volumes and driver mounts cannot cover it. Podman mounts the persistent +workspace volume there and copies existing image-directory files into a new +volume. OpenShell preserves their permissions and requires the final non-root +user to be able to write there. Kubernetes and MicroVM continue to use +`/sandbox`. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 3e0f70ad75..f18f1bb8b1 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -282,7 +282,7 @@ Common findings: - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default image workdir. - Docker also rejects an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. - A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the reserved paths named in the error. -- Podman rejects driver-configured mounts that cover the workdir but leaves image-declared volumes to Podman. The workspace volume keeps files copied from the image without changing their permissions; ensure the sandbox user can write the resulting directory. +- Podman also rejects an image volume or driver mount that covers the workdir. The workspace volume keeps files copied from the image without changing their permissions; ensure the sandbox user can write the resulting directory. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. From a7797839c6dc8ac3c48fb35046950d251b6cc2cf Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 18:52:20 -0700 Subject: [PATCH 10/20] fix(podman): admit image volumes under custom workdirs Signed-off-by: Matthew Grossman --- crates/openshell-driver-podman/README.md | 5 +- .../openshell-driver-podman/src/container.rs | 58 +++++++++-------- crates/openshell-driver-podman/src/driver.rs | 63 +++++++++++++++++++ e2e/rust/tests/podman_oci_identity.rs | 13 ++-- 4 files changed, 108 insertions(+), 31 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 48f50fa88e..ffd4044944 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -96,8 +96,9 @@ OpenShell reads `WORKDIR` from the workload image. If it is unset, `/`, or `/sandbox`, OpenShell uses its managed `/sandbox` workspace. A custom path must be absolute, with no `.` or `..` segments. It cannot overlap `/proc`, `/sys`, `/dev`, OpenShell-reserved paths, or the workload's private control and CA -mounts. Image volumes and driver mounts cannot cover the workspace or one of -its parents; mounts nested below it remain valid. +mounts. For a custom path, image volumes and driver mounts cannot cover the +workspace or one of its parents; mounts nested below it remain valid. Podman +creates a private volume for each image-declared path nested below it. For a custom path, Podman mounts a persistent workspace volume there. When the volume is first created, Podman copies any files already in that image directory diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 0ec102e422..1161b940af 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -225,13 +225,14 @@ pub struct ResolvedPodmanImage { pub(crate) oci_user: String, pub(crate) environment: Vec, pub(crate) workspace_root: String, + image_volume_targets: Vec, } impl ResolvedPodmanImage { /// Resolve the image metadata and reject: /// - a malformed or reserved working directory; - /// - an image volume with an invalid or reserved target, or one covering - /// the resolved workspace. + /// - for a custom working directory, an image volume with an invalid or + /// reserved target, or one covering the resolved workspace. pub fn from_inspect(inspected: &ImageInspect) -> Result { let image_config = inspected.config.as_ref(); let workspace_root = driver_mounts::resolve_oci_workspace_root( @@ -245,7 +246,10 @@ impl ResolvedPodmanImage { driver_mounts::validate_workspace_control_path(&workspace_root, control_path) .map_err(ComputeDriverError::Precondition)?; } - if let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) { + let mut image_volume_targets = Vec::new(); + if workspace_root != driver_mounts::DEFAULT_WORKSPACE_ROOT + && let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) + { for volume in volumes.keys() { validate_podman_mount_target(volume).map_err(|error| { ComputeDriverError::Precondition(format!( @@ -259,7 +263,9 @@ impl ResolvedPodmanImage { )) }, )?; + image_volume_targets.push(volume.clone()); } + image_volume_targets.sort(); } Ok(Self { id: inspected.id.clone(), @@ -268,6 +274,7 @@ impl ResolvedPodmanImage { .to_string(), environment: image_config.map_or_else(Vec::new, |config| config.env.clone()), workspace_root, + image_volume_targets, }) } @@ -1576,6 +1583,11 @@ pub fn build_isolation_specs( workload .labels .insert(crate::isolation::LABEL_ROLE.into(), "sandbox".into()); + workload.labels.insert( + "openshell.ai/private-image-volume-targets".into(), + serde_json::to_string(&input.image.image_volume_targets) + .map_err(|error| ComputeDriverError::Message(error.to_string()))?, + ); workload.env = BTreeMap::new(); workload.unsetenv = input .image @@ -2008,7 +2020,10 @@ mod tests { driver_mounts::DEFAULT_WORKSPACE_ROOT, ] ); - let custom_image = resolved_image("sha256:image", "1000:1001", "/workspace/project"); + let mut custom_image = resolved_image("sha256:image", "1000:1001", "/workspace/project"); + custom_image + .image_volume_targets + .push("/workspace/project/cache".into()); let custom_specs = build_isolation_specs(IsolationSpecInput { sandbox: &sandbox, config: &config, @@ -2024,6 +2039,14 @@ mod tests { }) .unwrap(); assert_eq!(custom_specs.workload.user, "1000:1001"); + assert_eq!( + custom_specs + .workload + .labels + .get("openshell.ai/private-image-volume-targets") + .map(String::as_str), + Some("[\"/workspace/project/cache\"]") + ); assert!(custom_specs.workload.cap_add.is_empty()); assert_eq!( custom_specs.workload.command, @@ -2125,28 +2148,13 @@ mod tests { let image = ResolvedPodmanImage::from_inspect(&inspect("/workspace/project/cache")) .expect("image volumes nested below the workspace remain valid"); assert_eq!(image.workspace_root, "/workspace/project"); - } - - #[test] - fn resolved_image_reuses_reserved_workdir_validation() { - let inspect = |working_dir: &str| ImageInspect { - id: "sha256:image".into(), - config: Some(ImageConfig { - working_dir: working_dir.into(), - ..Default::default() - }), - }; + assert_eq!(image.image_volume_targets, vec!["/workspace/project/cache"]); - assert!(ResolvedPodmanImage::from_inspect(&inspect("/opt/openshell/bin/project")).is_err()); - assert!(ResolvedPodmanImage::from_inspect(&inspect("/.openshell/channel")).is_err()); - assert!(ResolvedPodmanImage::from_inspect(&inspect("/etc/openshell/tls/client")).is_err()); - assert!(ResolvedPodmanImage::from_inspect(&inspect("/proc")).is_err()); - assert_eq!( - ResolvedPodmanImage::from_inspect(&inspect("/home/app")) - .unwrap() - .workspace_root, - "/home/app" - ); + let mut fallback = inspect("/etc/openshell"); + fallback.config.as_mut().unwrap().working_dir = "/sandbox".into(); + let fallback = ResolvedPodmanImage::from_inspect(&fallback) + .expect("existing /sandbox images keep their image-volume behavior"); + assert!(fallback.image_volume_targets.is_empty()); } fn json_struct(value: Value) -> prost_types::Struct { diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 4af454ec54..a28b3d271b 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -761,6 +761,10 @@ impl PodmanComputeDriver { .get(openshell_core::resource_admission::IDENTITIES_LABEL) .and_then(|value| serde_json::from_str(value).ok()) .ok_or_else(missing)?; + let anonymous_targets: Vec = labels + .get("openshell.ai/private-image-volume-targets") + .and_then(|value| serde_json::from_str(value).ok()) + .unwrap_or_default(); let mut actual = std::collections::BTreeMap::new(); for mount in mounts { match mount["Type"].as_str() { @@ -789,6 +793,16 @@ impl PodmanComputeDriver { if !owned || volume.driver != "local" || !volume.options.is_empty() { return Err(missing()); } + } else if !expected.contains_key(name) + && mount["Destination"].as_str().is_some_and(|destination| { + anonymous_targets.iter().any(|target| target == destination) + }) + { + if volume.driver != "local" || !volume.options.is_empty() { + return Err(ComputeDriverError::Precondition( + "image-private volume backing changed".into(), + )); + } } else { actual.insert(name.to_string(), volume.admission_identity()); self.config @@ -3098,6 +3112,55 @@ mod tests { } } + #[tokio::test] + async fn admission_accepts_image_declared_volume_below_custom_workdir() { + let (socket, requests, handle) = spawn_podman_stub( + "image-volume-admission", + vec![ + StubResponse::new( + StatusCode::OK, + serde_json::json!({ + "Id": "workload-1", + "Name": "workload-1", + "State": {"Status": "created", "Running": false}, + "Config": {"Labels": { + "openshell.ai/sandbox-workspace": "team-a", + "openshell.ai/sandbox-id": "sandbox-1", + "openshell.ai/caller-driver-config-used": "false", + "openshell.ai/resource-admission-identities": "{}", + "openshell.ai/private-image-volume-targets": "[\"/home/app/project/cache\"]" + }}, + "Mounts": [{ + "Type": "volume", + "Name": "anonymous-1", + "Destination": "/home/app/project/cache" + }] + }) + .to_string(), + ), + StubResponse::new( + StatusCode::OK, + serde_json::json!({ + "Name": "anonymous-1", "Driver": "local", "Options": {}, "Labels": {} + }) + .to_string(), + ), + ], + ); + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + socket_path: Some(socket.clone()), + ..Default::default() + }); + + driver + .admit_container_resources("workload-1") + .await + .expect("a local image-declared volume is private to the workload"); + handle.await.unwrap(); + assert_eq!(requests.lock().unwrap().len(), 2); + let _ = fs::remove_file(socket); + } + #[tokio::test] async fn admission_driver_config_denial_does_not_contact_podman() { for enabled in [true, false] { diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index 5454401851..53725e9b4f 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -56,11 +56,12 @@ impl ImageGuard { format!( r"FROM {BASE_IMAGE} USER 0:0 -RUN mkdir -p /home/app/project && \ - chown {OCI_UID}:{OCI_GID} /home/app /home/app/project && \ - chmod 0700 /home/app /home/app/project +RUN mkdir -p /home/app/project/cache && \ + chown {OCI_UID}:{OCI_GID} /home/app /home/app/project /home/app/project/cache && \ + chmod 0700 /home/app /home/app/project /home/app/project/cache WORKDIR /home/app/project RUN printf root-owned > root-owned.txt && chown {OCI_UID}:{OCI_GID} . +VOLUME /home/app/project/cache USER {OCI_UID}:{OCI_GID} " ), @@ -204,8 +205,9 @@ async fn podman_uses_oci_identity_workspace_copy_up_and_inspected_image_id() { "set -eu; \ test \"$(pwd -P)\" = /home/app/project; \ test \"$HOME\" = /home/app/project; \ - test \"$(stat -c %u:%g root-owned.txt)\" = 0:0; \ + test \"$(cat root-owned.txt)\" = root-owned; \ touch direct-workspace-write; \ + touch cache/from-create; \ printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; \ echo podman-oci-identity-ready; sleep infinity", ], @@ -228,7 +230,9 @@ async fn podman_uses_oci_identity_workspace_copy_up_and_inspected_image_id() { test \"$(id -u):$(id -g)\" = 2345:2346; \ test \"$(pwd -P)\" = /home/app/project; \ test -f direct-workspace-write; \ + test -f cache/from-create; \ touch ssh-workspace-write; \ + touch cache/from-ssh; \ echo podman-ssh-identity-ok", ]) .await @@ -314,6 +318,7 @@ async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, contai assert!(!mounts.contains("/etc/openshell/tls")); assert!(!mounts.contains("/.openshell/supervisor")); assert!(mounts.lines().any(|path| path == "/home/app/project")); + assert!(mounts.lines().any(|path| path == "/home/app/project/cache")); assert!(!mounts.lines().any(|path| path == "/sandbox")); let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/runtime-descriptor.json"]).await.expect("workload cannot read either control credential set"); assert!(posture.contains("0000000000000000")); From 2405282a3822c2abb2b19ec1867b9fda1bf54bd6 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 20:19:21 -0700 Subject: [PATCH 11/20] test(podman): run OCI workdir E2E in CI Signed-off-by: Matthew Grossman --- e2e/rust/e2e-podman.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index bbbcfe6fa9..4d23fd831f 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -38,6 +38,7 @@ PODMAN_CI_TESTS=( podman_corporate_proxy podman_gateway_start podman_host_gateway + podman_oci_identity provider_token_exchange ) From fc4d8bdc615e91afeb87737e60d44852acd8ada3 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 21:04:54 -0700 Subject: [PATCH 12/20] refactor(driver): share mount target control-path validation Signed-off-by: Matthew Grossman --- crates/openshell-core/src/driver_mounts.rs | 33 +++++++++++++++++ crates/openshell-driver-docker/src/lib.rs | 15 +++++--- .../openshell-driver-podman/src/container.rs | 36 ++++++++----------- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index b1a3049882..2cce47c63f 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -98,6 +98,18 @@ pub fn validate_container_mount_target(target: &str) -> Result<(), String> { Ok(()) } +/// Validate a mount target against shared and driver-specific control paths. +pub fn validate_container_mount_target_with_control_paths( + target: &str, + control_paths: &[&str], +) -> Result<(), String> { + validate_container_mount_target(target)?; + for control_path in control_paths { + validate_mount_control_path(target, control_path)?; + } + Ok(()) +} + /// Resolve an OCI image working directory to the internal workspace root used /// by local container drivers. /// @@ -360,6 +372,27 @@ mod tests { validate_mount_control_path("/custom-other", "/custom/ssh.sock").unwrap(); } + #[test] + fn container_target_checks_shared_and_driver_control_paths() { + let control_paths = &["/.openshell/channel"]; + assert!( + validate_container_mount_target_with_control_paths( + "/etc/openshell/tls/client", + control_paths, + ) + .is_err() + ); + assert!( + validate_container_mount_target_with_control_paths( + "/.openshell/channel/sandbox", + control_paths, + ) + .is_err() + ); + validate_container_mount_target_with_control_paths("/workspace/cache", control_paths) + .unwrap(); + } + #[test] fn workspace_rejects_malformed_runtime_control_paths() { for control_path in [ diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 1f4c91f5e4..99249fb0eb 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5657,7 +5657,11 @@ fn build_container_create_body_for_image( driver_mounts::validate_workspace_control_path(&workspace_root, BOUNDARY_MOUNT_PATH) .map_err(Status::failed_precondition)?; for volume in &image.volumes { - driver_mounts::validate_container_mount_target(volume).map_err(|error| { + driver_mounts::validate_container_mount_target_with_control_paths( + volume, + &[BOUNDARY_MOUNT_PATH], + ) + .map_err(|error| { Status::failed_precondition(format!( "invalid image-declared volume '{volume}': {error}" )) @@ -5667,8 +5671,6 @@ fn build_container_create_body_for_image( "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" )) })?; - driver_mounts::validate_mount_control_path(volume, BOUNDARY_MOUNT_PATH) - .map_err(Status::failed_precondition)?; } for mount in &driver_config.mounts { let target = match mount { @@ -5679,8 +5681,11 @@ fn build_container_create_body_for_image( }; driver_mounts::validate_workspace_mount_target(target, &workspace_root) .map_err(Status::failed_precondition)?; - driver_mounts::validate_mount_control_path(target, BOUNDARY_MOUNT_PATH) - .map_err(Status::failed_precondition)?; + driver_mounts::validate_container_mount_target_with_control_paths( + target, + &[BOUNDARY_MOUNT_PATH], + ) + .map_err(Status::failed_precondition)?; } let mut user_mounts = docker_driver_mounts(driver_config)?; user_mounts.push(Mount { diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 1161b940af..9454ffeadd 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -6,7 +6,9 @@ use crate::client::ImageInspect; use crate::config::PodmanComputeConfig; use openshell_core::ComputeDriverError; -use openshell_core::driver_mounts::SelinuxLabel; +use openshell_core::driver_mounts::{ + SelinuxLabel, validate_container_mount_target_with_control_paths as validate_mount_target, +}; #[cfg(test)] use openshell_core::gpu::{driver_gpu_requirements, validate_specific_gpu_device_request}; use openshell_core::proto::compute::v1::{DriverSandbox, DriverSandboxTemplate}; @@ -50,6 +52,10 @@ const CONTAINER_PREFIX: &str = "openshell-"; /// Volume name prefix. const VOLUME_PREFIX: &str = "openshell-sandbox-"; +const WORKLOAD_CONTROL_PATHS: &[&str] = &[ + "/.openshell", + openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, +]; /// Secret name prefix for per-sandbox gateway JWTs. const TOKEN_SECRET_PREFIX: &str = "openshell-token-"; @@ -239,10 +245,7 @@ impl ResolvedPodmanImage { image_config.map_or("", |config| config.working_dir.as_str()), ) .map_err(ComputeDriverError::Precondition)?; - for control_path in [ - "/.openshell", - openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, - ] { + for control_path in WORKLOAD_CONTROL_PATHS { driver_mounts::validate_workspace_control_path(&workspace_root, control_path) .map_err(ComputeDriverError::Precondition)?; } @@ -251,7 +254,7 @@ impl ResolvedPodmanImage { && let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) { for volume in volumes.keys() { - validate_podman_mount_target(volume).map_err(|error| { + validate_mount_target(volume, WORKLOAD_CONTROL_PATHS).map_err(|error| { ComputeDriverError::Precondition(format!( "invalid image-declared volume '{volume}': {error}" )) @@ -840,17 +843,6 @@ pub fn podman_driver_image_mount_sources( .collect()) } -fn validate_podman_mount_target(target: &str) -> Result<(), String> { - driver_mounts::validate_container_mount_target(target)?; - for control_path in [ - "/.openshell", - openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, - ] { - driver_mounts::validate_mount_control_path(target, control_path)?; - } - Ok(()) -} - fn podman_user_mounts( sandbox: &DriverSandbox, enable_bind_mounts: bool, @@ -882,7 +874,7 @@ fn podman_user_mounts( None => {} } driver_mounts::validate_absolute_mount_source(&source, "bind source")?; - validate_podman_mount_target(&target)?; + validate_mount_target(&target, WORKLOAD_CONTROL_PATHS)?; result.mounts.push(Mount { kind: "bind".into(), source, @@ -898,7 +890,7 @@ fn podman_user_mounts( } => { reject_subpath(subpath.as_deref(), "podman volume mounts")?; driver_mounts::validate_mount_source(&source, "volume source")?; - validate_podman_mount_target(&target)?; + validate_mount_target(&target, WORKLOAD_CONTROL_PATHS)?; result.volumes.push(NamedVolume { name: source, dest: target, @@ -924,7 +916,7 @@ fn podman_user_mounts( { options.push(format!("mode={mode:o}")); } - validate_podman_mount_target(&target)?; + validate_mount_target(&target, WORKLOAD_CONTROL_PATHS)?; result.mounts.push(Mount { kind: "tmpfs".into(), source: "tmpfs".into(), @@ -940,7 +932,7 @@ fn podman_user_mounts( } => { reject_subpath(subpath.as_deref(), "podman image mounts")?; driver_mounts::validate_mount_source(&source, "image source")?; - validate_podman_mount_target(&target)?; + validate_mount_target(&target, WORKLOAD_CONTROL_PATHS)?; result.image_volumes.push(ImageVolume { source, destination: target, @@ -1015,7 +1007,7 @@ fn validate_podman_driver_mounts( target } }; - validate_podman_mount_target(target)?; + validate_mount_target(target, WORKLOAD_CONTROL_PATHS)?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(format!( From 7d11c2c876ba09f127ccea5ba31b972bb92f8d74 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 21:43:51 -0700 Subject: [PATCH 13/20] refactor(podman): call shared mount validator explicitly Signed-off-by: Matthew Grossman --- .../openshell-driver-podman/src/container.rs | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 9454ffeadd..6b4dd78200 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -6,9 +6,7 @@ use crate::client::ImageInspect; use crate::config::PodmanComputeConfig; use openshell_core::ComputeDriverError; -use openshell_core::driver_mounts::{ - SelinuxLabel, validate_container_mount_target_with_control_paths as validate_mount_target, -}; +use openshell_core::driver_mounts::SelinuxLabel; #[cfg(test)] use openshell_core::gpu::{driver_gpu_requirements, validate_specific_gpu_device_request}; use openshell_core::proto::compute::v1::{DriverSandbox, DriverSandboxTemplate}; @@ -254,7 +252,11 @@ impl ResolvedPodmanImage { && let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) { for volume in volumes.keys() { - validate_mount_target(volume, WORKLOAD_CONTROL_PATHS).map_err(|error| { + driver_mounts::validate_container_mount_target_with_control_paths( + volume, + WORKLOAD_CONTROL_PATHS, + ) + .map_err(|error| { ComputeDriverError::Precondition(format!( "invalid image-declared volume '{volume}': {error}" )) @@ -874,7 +876,10 @@ fn podman_user_mounts( None => {} } driver_mounts::validate_absolute_mount_source(&source, "bind source")?; - validate_mount_target(&target, WORKLOAD_CONTROL_PATHS)?; + driver_mounts::validate_container_mount_target_with_control_paths( + &target, + WORKLOAD_CONTROL_PATHS, + )?; result.mounts.push(Mount { kind: "bind".into(), source, @@ -890,7 +895,10 @@ fn podman_user_mounts( } => { reject_subpath(subpath.as_deref(), "podman volume mounts")?; driver_mounts::validate_mount_source(&source, "volume source")?; - validate_mount_target(&target, WORKLOAD_CONTROL_PATHS)?; + driver_mounts::validate_container_mount_target_with_control_paths( + &target, + WORKLOAD_CONTROL_PATHS, + )?; result.volumes.push(NamedVolume { name: source, dest: target, @@ -916,7 +924,10 @@ fn podman_user_mounts( { options.push(format!("mode={mode:o}")); } - validate_mount_target(&target, WORKLOAD_CONTROL_PATHS)?; + driver_mounts::validate_container_mount_target_with_control_paths( + &target, + WORKLOAD_CONTROL_PATHS, + )?; result.mounts.push(Mount { kind: "tmpfs".into(), source: "tmpfs".into(), @@ -932,7 +943,10 @@ fn podman_user_mounts( } => { reject_subpath(subpath.as_deref(), "podman image mounts")?; driver_mounts::validate_mount_source(&source, "image source")?; - validate_mount_target(&target, WORKLOAD_CONTROL_PATHS)?; + driver_mounts::validate_container_mount_target_with_control_paths( + &target, + WORKLOAD_CONTROL_PATHS, + )?; result.image_volumes.push(ImageVolume { source, destination: target, @@ -1007,7 +1021,10 @@ fn validate_podman_driver_mounts( target } }; - validate_mount_target(target, WORKLOAD_CONTROL_PATHS)?; + driver_mounts::validate_container_mount_target_with_control_paths( + target, + WORKLOAD_CONTROL_PATHS, + )?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(format!( From be4790c32bbe1b26d2fc4a0abfc78608b20fff31 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 28 Sep 2026 21:49:33 -0700 Subject: [PATCH 14/20] refactor(driver): share image-volume admission label Signed-off-by: Matthew Grossman --- .../openshell-core/src/resource_admission.rs | 1 + crates/openshell-driver-docker/src/lib.rs | 4 ++-- .../openshell-driver-podman/src/container.rs | 20 +++++++++---------- crates/openshell-driver-podman/src/driver.rs | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/crates/openshell-core/src/resource_admission.rs b/crates/openshell-core/src/resource_admission.rs index 7d20c405f4..b0229e0142 100644 --- a/crates/openshell-core/src/resource_admission.rs +++ b/crates/openshell-core/src/resource_admission.rs @@ -93,6 +93,7 @@ impl std::str::FromStr for DriverAdmissionConfig { /// Reserved driver-owned runtime metadata; caller labels must never override it. pub const CONFIG_USED_LABEL: &str = "openshell.ai/caller-driver-config-used"; pub const IDENTITIES_LABEL: &str = "openshell.ai/resource-admission-identities"; +pub const PRIVATE_IMAGE_VOLUME_TARGETS_LABEL: &str = "openshell.ai/private-image-volume-targets"; pub fn check_config_provenance(allowed: bool, recorded: Option<&str>) -> Result<(), tonic::Status> { match recorded { diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 99249fb0eb..831acd3f3f 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -1215,7 +1215,7 @@ impl DockerComputeDriver { .ok_or_else(|| Status::failed_precondition("sandbox lacks resource identity record"))?; let mut actual = std::collections::BTreeMap::new(); let anonymous_targets: Vec = labels - .get("openshell.ai/private-image-volume-targets") + .get(openshell_core::resource_admission::PRIVATE_IMAGE_VOLUME_TARGETS_LABEL) .and_then(|value| serde_json::from_str(value).ok()) .unwrap_or_default(); for mount in container.mounts.as_deref().unwrap_or_default() { @@ -5709,7 +5709,7 @@ fn build_container_create_body_for_image( }); let mut labels = template.labels.clone(); labels.insert( - "openshell.ai/private-image-volume-targets".into(), + openshell_core::resource_admission::PRIVATE_IMAGE_VOLUME_TARGETS_LABEL.into(), serde_json::to_string(&image.volumes) .map_err(|error| Status::internal(error.to_string()))?, ); diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 6b4dd78200..106331c311 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -50,7 +50,7 @@ const CONTAINER_PREFIX: &str = "openshell-"; /// Volume name prefix. const VOLUME_PREFIX: &str = "openshell-sandbox-"; -const WORKLOAD_CONTROL_PATHS: &[&str] = &[ +const PODMAN_WORKLOAD_CONTROL_PATHS: &[&str] = &[ "/.openshell", openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT, ]; @@ -243,7 +243,7 @@ impl ResolvedPodmanImage { image_config.map_or("", |config| config.working_dir.as_str()), ) .map_err(ComputeDriverError::Precondition)?; - for control_path in WORKLOAD_CONTROL_PATHS { + for control_path in PODMAN_WORKLOAD_CONTROL_PATHS { driver_mounts::validate_workspace_control_path(&workspace_root, control_path) .map_err(ComputeDriverError::Precondition)?; } @@ -254,7 +254,7 @@ impl ResolvedPodmanImage { for volume in volumes.keys() { driver_mounts::validate_container_mount_target_with_control_paths( volume, - WORKLOAD_CONTROL_PATHS, + PODMAN_WORKLOAD_CONTROL_PATHS, ) .map_err(|error| { ComputeDriverError::Precondition(format!( @@ -878,7 +878,7 @@ fn podman_user_mounts( driver_mounts::validate_absolute_mount_source(&source, "bind source")?; driver_mounts::validate_container_mount_target_with_control_paths( &target, - WORKLOAD_CONTROL_PATHS, + PODMAN_WORKLOAD_CONTROL_PATHS, )?; result.mounts.push(Mount { kind: "bind".into(), @@ -897,7 +897,7 @@ fn podman_user_mounts( driver_mounts::validate_mount_source(&source, "volume source")?; driver_mounts::validate_container_mount_target_with_control_paths( &target, - WORKLOAD_CONTROL_PATHS, + PODMAN_WORKLOAD_CONTROL_PATHS, )?; result.volumes.push(NamedVolume { name: source, @@ -926,7 +926,7 @@ fn podman_user_mounts( } driver_mounts::validate_container_mount_target_with_control_paths( &target, - WORKLOAD_CONTROL_PATHS, + PODMAN_WORKLOAD_CONTROL_PATHS, )?; result.mounts.push(Mount { kind: "tmpfs".into(), @@ -945,7 +945,7 @@ fn podman_user_mounts( driver_mounts::validate_mount_source(&source, "image source")?; driver_mounts::validate_container_mount_target_with_control_paths( &target, - WORKLOAD_CONTROL_PATHS, + PODMAN_WORKLOAD_CONTROL_PATHS, )?; result.image_volumes.push(ImageVolume { source, @@ -1023,7 +1023,7 @@ fn validate_podman_driver_mounts( }; driver_mounts::validate_container_mount_target_with_control_paths( target, - WORKLOAD_CONTROL_PATHS, + PODMAN_WORKLOAD_CONTROL_PATHS, )?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { @@ -1593,7 +1593,7 @@ pub fn build_isolation_specs( .labels .insert(crate::isolation::LABEL_ROLE.into(), "sandbox".into()); workload.labels.insert( - "openshell.ai/private-image-volume-targets".into(), + openshell_core::resource_admission::PRIVATE_IMAGE_VOLUME_TARGETS_LABEL.into(), serde_json::to_string(&input.image.image_volume_targets) .map_err(|error| ComputeDriverError::Message(error.to_string()))?, ); @@ -2052,7 +2052,7 @@ mod tests { custom_specs .workload .labels - .get("openshell.ai/private-image-volume-targets") + .get(openshell_core::resource_admission::PRIVATE_IMAGE_VOLUME_TARGETS_LABEL) .map(String::as_str), Some("[\"/workspace/project/cache\"]") ); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index a28b3d271b..20b7d90c37 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -762,7 +762,7 @@ impl PodmanComputeDriver { .and_then(|value| serde_json::from_str(value).ok()) .ok_or_else(missing)?; let anonymous_targets: Vec = labels - .get("openshell.ai/private-image-volume-targets") + .get(openshell_core::resource_admission::PRIVATE_IMAGE_VOLUME_TARGETS_LABEL) .and_then(|value| serde_json::from_str(value).ok()) .unwrap_or_default(); let mut actual = std::collections::BTreeMap::new(); From 93c61b0a8646ab1f44612e2129b9681de64595ba Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 29 Sep 2026 09:40:38 -0700 Subject: [PATCH 15/20] docs(podman): avoid promising workspace as HOME Signed-off-by: Matthew Grossman --- crates/openshell-driver-podman/README.md | 2 +- e2e/rust/tests/podman_oci_identity.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index ffd4044944..0c8064df29 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -104,7 +104,7 @@ For a custom path, Podman mounts a persistent workspace volume there. When the volume is first created, Podman copies any files already in that image directory into it. OpenShell keeps their ownership and permissions, starts as the final non-root user, and rejects the image if that user cannot reach and write the -directory. Agent commands use the path as their working directory and `HOME`; +directory. Agent commands use the path as their working directory; `filesystem.include_workdir` grants access to it when enabled. For `/sandbox`, OpenShell prepares the managed workspace before switching to diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index 53725e9b4f..dbe5f26709 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -204,7 +204,6 @@ async fn podman_uses_oci_identity_workspace_copy_up_and_inspected_image_id() { "-c", "set -eu; \ test \"$(pwd -P)\" = /home/app/project; \ - test \"$HOME\" = /home/app/project; \ test \"$(cat root-owned.txt)\" = root-owned; \ touch direct-workspace-write; \ touch cache/from-create; \ From 272a2b3269b604c56b8fd5ed35e65d76fe02f81a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 29 Sep 2026 10:50:35 -0700 Subject: [PATCH 16/20] fix(podman): use container filesystem for image workdirs Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 8 +++--- crates/openshell-driver-podman/README.md | 28 +++++++++---------- .../openshell-driver-podman/src/container.rs | 20 ++++++------- crates/openshell-driver-podman/src/driver.rs | 15 ++++++---- docs/how-it-works/sandboxes/runtimes.mdx | 9 +++--- e2e/rust/tests/podman_oci_identity.rs | 8 +++--- 6 files changed, 43 insertions(+), 45 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 95736400bc..c64778494e 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -430,10 +430,10 @@ resolved identity. Podman reads the image user and working directory from one pinned image. Empty, `/`, and explicit `/sandbox` values use the managed `/sandbox` workspace. A custom path must be absolute, normalized, and outside system and OpenShell -reserved paths. Image and driver mounts cannot cover the workspace. Podman -mounts the persistent workspace volume there and copies any existing image -files into it on first use. OpenShell preserves their ownership and permissions; -the final non-root user must be able to write the resulting workspace. +reserved paths. Image and driver mounts cannot cover the workspace. Custom +paths use the image's container filesystem directly; the final non-root user +must be able to write the existing directory. The managed `/sandbox` fallback +uses a driver-owned workspace volume. Kubernetes and VM use `/sandbox`. ### Executable Identity Binding diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 0c8064df29..4596377f26 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -37,13 +37,12 @@ The supervisor joins the workload's **user namespace only** to preserve UID/GID mapping for shared-volume access. PID, mount, and network namespaces remain separate. The channel volume uses shared SELinux relabeling (`:z`). -Before starting either container, the driver uploads volume-relative archives -directly to the channel and workspace volume destinations. A rootfs upload on a -stopped Podman container does not populate nested named volumes. Restart restores -only the channel bootstrap into the existing channel volume, preserving the -workspace. The workload starts before the supervisor so its user namespace exists -when the supervisor joins it; a stopped supervisor resolves that namespace again -on its next start. +Before starting either container, the driver uploads bootstrap files to the +channel volume. For managed `/sandbox`, it also prepares the workspace volume; +custom image workspaces need no upload. Restart restores only the channel +bootstrap, preserving the workspace. The workload starts before the supervisor +so its user namespace exists when the supervisor joins it; a stopped supervisor +resolves that namespace again on its next start. The runtime must pass the sandbox's unprivileged enforcement probe, including nested seccomp notification and Landlock. Unsupported runtime defaults fail @@ -100,16 +99,15 @@ mounts. For a custom path, image volumes and driver mounts cannot cover the workspace or one of its parents; mounts nested below it remain valid. Podman creates a private volume for each image-declared path nested below it. -For a custom path, Podman mounts a persistent workspace volume there. When the -volume is first created, Podman copies any files already in that image directory -into it. OpenShell keeps their ownership and permissions, starts as the final -non-root user, and rejects the image if that user cannot reach and write the -directory. Agent commands use the path as their working directory; +For a custom path, Podman uses the image's container filesystem directly. +OpenShell keeps the image directory's ownership and permissions, starts as the +final non-root user, and rejects the image if that user cannot reach and write +the directory. Agent commands use the path as their working directory; `filesystem.include_workdir` grants access to it when enabled. -For `/sandbox`, OpenShell prepares the managed workspace before switching to -the non-root user. The separate supervisor receives the path but does not mount -the workspace. Test custom images with the Podman configuration you will use. +For `/sandbox`, OpenShell creates a workspace volume and prepares it before +switching to the non-root user. The separate supervisor receives the path but +does not mount the workspace. ## Lifecycle and readiness diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 106331c311..f87c7362ef 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1257,11 +1257,14 @@ fn build_base_spec( let mut networks = BTreeMap::new(); networks.insert(config.network_name.clone(), NetworkAttachment {}); - let mut volumes = vec![NamedVolume { - name: vol, - dest: image.workspace_root.clone(), - options: vec!["rw".into()], - }]; + let mut volumes = Vec::new(); + if image.uses_managed_workspace() { + volumes.push(NamedVolume { + name: vol, + dest: image.workspace_root.clone(), + options: vec!["rw".into()], + }); + } volumes.extend(user_mounts.volumes); let mut image_volumes = if supervisor_bin_path.is_some() { @@ -2303,12 +2306,7 @@ mod tests { container["command"], serde_json::json!(["--workdir", "/workspace/project"]) ); - assert!(container["volumes"].as_array().is_some_and(|volumes| { - volumes.iter().any(|volume| { - volume["name"].as_str() == Some("openshell-sandbox-test-id-workspace") - && volume["dest"].as_str() == Some("/workspace/project") - }) - })); + assert!(container["volumes"].as_array().is_some_and(Vec::is_empty)); assert_eq!( container["env"][openshell_core::sandbox_env::OCI_IMAGE_USER].as_str(), Some("app:staff") diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 20b7d90c37..130b17cb17 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1004,6 +1004,7 @@ impl PodmanComputeDriver { otel.status_code = tracing::field::Empty, )) .await?; + let managed_workspace = resolved_image.uses_managed_workspace(); // Fail closed on a missing/unreadable corporate proxy CA bundle before // creating any resources, so the operator gets a clear error @@ -1042,14 +1043,16 @@ impl PodmanComputeDriver { )); } - // Create the workspace volume and per-sandbox runtime files. + // Create the managed workspace volume, if needed, and runtime files. let (resolver_secret_name, token_secret_name, proxy_auth_secret_name) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { - self.client - .create_owned_volume(&vol_name, &sandbox.id, &sandbox.workspace) - .await - .map_err(ComputeDriverError::from)?; + if managed_workspace { + self.client + .create_owned_volume(&vol_name, &sandbox.id, &sandbox.workspace) + .await + .map_err(ComputeDriverError::from)?; + } let resolver_secret_name = match create_sandbox_resolver_secret(&self.client, &sandbox.id).await { Ok(name) => name, @@ -1231,7 +1234,7 @@ impl PodmanComputeDriver { archives.channel, ) .await?; - if resolved_image.uses_managed_workspace() { + if managed_workspace { self.client .copy_to_container( &workload_id, diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index ead8b55179..e03054d651 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -272,8 +272,7 @@ On Docker, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR` Podman also uses the image's `WORKDIR` as the workspace, falling back to `/sandbox` for an empty, `/`, or `/sandbox` value. A custom path must be an absolute, normalized path outside system and OpenShell-reserved paths. -Image volumes and driver mounts cannot cover it. Podman mounts the persistent -workspace volume there and copies existing image-directory files into a new -volume. OpenShell preserves their permissions and requires the final non-root -user to be able to write there. Kubernetes and MicroVM continue to use -`/sandbox`. +Image volumes and driver mounts cannot cover it. Podman uses the image's +container filesystem directly and requires the final non-root user to be able +to write the existing directory. The managed `/sandbox` fallback uses a +workspace volume. Kubernetes and MicroVM continue to use `/sandbox`. diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index dbe5f26709..78a57a4a81 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -4,10 +4,10 @@ #![cfg(feature = "e2e-podman")] //! Podman-specific E2E coverage for OCI identity/workspace inspection, -//! workspace-volume copy-up, and immutable-image launch. +//! direct image-workspace access, and immutable-image launch. //! //! The test builds an image through the selected Podman engine, creates a -//! sandbox from its mutable tag, and verifies the child identity, copied image +//! sandbox from its mutable tag, and verifies the child identity, image //! content, workspace placement, and image ID recorded on the real sandbox //! container. @@ -184,7 +184,7 @@ fn normalized_image_id(image_id: &str) -> &str { } #[tokio::test] -async fn podman_uses_oci_identity_workspace_copy_up_and_inspected_image_id() { +async fn podman_uses_oci_identity_image_workdir_and_inspected_image_id() { if !is_e2e_driver("podman") { eprintln!("Skipping Podman OCI identity test: e2e driver is not podman"); return; @@ -316,7 +316,7 @@ async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, contai .unwrap(); assert!(!mounts.contains("/etc/openshell/tls")); assert!(!mounts.contains("/.openshell/supervisor")); - assert!(mounts.lines().any(|path| path == "/home/app/project")); + assert!(!mounts.lines().any(|path| path == "/home/app/project")); assert!(mounts.lines().any(|path| path == "/home/app/project/cache")); assert!(!mounts.lines().any(|path| path == "/sandbox")); let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/runtime-descriptor.json"]).await.expect("workload cannot read either control credential set"); From ba9645e4759756e3cc2d6035931f690f3f8935c8 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 29 Sep 2026 12:28:34 -0700 Subject: [PATCH 17/20] docs(podman): clarify custom workdir storage Signed-off-by: Matthew Grossman --- skills/debug-openshell-cluster/SKILL.md | 2 +- skills/openshell-cli/SKILL.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index f18f1bb8b1..0d614d90d3 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -282,7 +282,7 @@ Common findings: - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default image workdir. - Docker also rejects an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. - A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the reserved paths named in the error. -- Podman also rejects an image volume or driver mount that covers the workdir. The workspace volume keeps files copied from the image without changing their permissions; ensure the sandbox user can write the resulting directory. +- Podman also rejects an image volume or driver mount that covers a custom workdir. A custom workdir uses the image's container filesystem directly, without a workspace volume; ensure the sandbox user can write the existing directory. Only the `/sandbox` fallback uses a workspace volume. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index c77a6e487c..4f729f1133 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -681,7 +681,8 @@ files, mounted volumes, or devices. Podman gateways use a normalized absolute OCI `WORKDIR` as the workspace. Empty, `/`, and explicit `/sandbox` declarations use the managed `/sandbox` fallback. For a custom path, the final identity must be able to traverse and -write the directory in the persistent volume. +write the existing directory in the container filesystem. A custom path does +not use a workspace volume; only the managed `/sandbox` fallback does. ### Forward ports From befdfebf13342518bf43d7a357292298dc9a3ce8 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 29 Sep 2026 12:45:42 -0700 Subject: [PATCH 18/20] refactor(podman): trim OCI workdir change set Drop image-volume admission (split to a follow-up), the Docker and core mount-validation refactor, redundant per-arm Podman mount checks, and unrelated doc and test churn. Keep image VOLUME masking validation for custom workdirs. Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 51 +++---- crates/openshell-core/src/driver_mounts.rs | 33 ----- .../openshell-core/src/resource_admission.rs | 1 - crates/openshell-driver-docker/src/lib.rs | 19 +-- crates/openshell-driver-podman/README.md | 39 +++--- .../openshell-driver-podman/src/container.rs | 128 ++++++------------ crates/openshell-driver-podman/src/driver.rs | 69 +--------- docs/how-it-works/sandboxes/runtimes.mdx | 10 +- e2e/rust/tests/podman_oci_identity.rs | 14 +- skills/debug-openshell-cluster/SKILL.md | 5 +- skills/openshell-cli/SKILL.md | 6 - 11 files changed, 93 insertions(+), 282 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index c64778494e..5b7f4bb5c7 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -9,12 +9,10 @@ Podman provisions a paired workload and supervisor container using its native libpod API. The workload uses `network=none`; the external supervisor alone joins the configured network. A per-sandbox named volume carries their mutually authenticated gRPC Unix socket, with supervisor credentials kept in its separate -filesystem. The supervisor and final sandbox runtime run as the resolved non-root -identity with all capabilities dropped. Only the managed `/sandbox` fallback uses -a trusted root bootstrap to prepare its driver-owned workspace before dropping -irreversibly to that identity. The containers do not share PID, mount, or network -namespaces. Podman owns paired lifecycle and health; the common protocol owns -process, identity, TCP, DNS, and forwarding semantics. +filesystem. Both containers run as the resolved non-root identity with all +capabilities dropped. They share only a user namespace for volume ownership, +not PID, mount, or network namespaces. Podman owns paired lifecycle and health; +the common protocol owns process, identity, TCP, DNS, and forwarding semantics. ## Driver Contract @@ -307,7 +305,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| | Docker | Local development with Docker available. | Capability-free workload container. | Uses `network_mode=none`; a separate capability-free supervisor container mediates egress and access over a private daemon-local Unix socket volume. | -| Podman | Local development with Podman available. | Capability-free workload container. | Uses `network=none`; a separate capability-free supervisor container mediates egress and access over a private Unix socket volume. | +| Podman | Existing rootless driver. | Container. | Not converted by this isolation stack. | | Kubernetes | Cluster deployment through Helm. | Capability-free sandbox Pod. | Always creates a namespace-wide empty-egress workload NetworkPolicy and a separate capability-free supervisor Pod over mutually authenticated TLS. It requires an enforcing CNI and trusted sandbox namespace; the Kubernetes API does not attest policy enforcement. | | VM | Experimental microVM isolation. | Per-sandbox libkrun or QEMU VM. | The NIC-less guest runs `openshell-sandbox` as PID 1; host `openshell-supervisor` owns gateway networking and reaches the guest over vsock. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | @@ -392,7 +390,7 @@ Drivers deliver the two binaries to separate trust domains: | Runtime | Delivery model | |---|---| | Docker | A digest-pinned daemon-local volume supplies `openshell-sandbox`; the companion image runs `openshell-supervisor`. | -| Podman | The driver pins `sandbox_runtime_image` and `supervisor_image` to image IDs. The former supplies `openshell-sandbox`; the latter is passed as the companion container's image and runs `openshell-supervisor`. | +| Podman | Existing driver behavior; not converted by this stack. | | Kubernetes | A non-root init container stages `openshell-sandbox` into a memory volume; a directly managed Pod runs `openshell-supervisor`. | | VM | `openshell-sandbox` is embedded in the guest rootfs; a separately digest-checked native `openshell-supervisor` runs on the host. | | Extension | Defined by the out-of-tree driver. | @@ -408,33 +406,24 @@ The gateway preserves whether each policy process field was omitted and passes the admitted selectors to the driver. The driver resolves one exact UID, GID, and supplementary-group set before creating the immutable workload: -- Docker pins the image ID, resolves policy selectors against the image's - `/etc/passwd` and `/etc/group`, and validates its OCI working directory. -- Podman pins the image ID, resolves policy selectors against the image's - `/etc/passwd` and `/etc/group`, and validates its OCI working directory. +- Docker and Podman pin the image ID, resolve policy selectors against the + image's `/etc/passwd` and `/etc/group`, and validate its OCI working + directory. - Kubernetes uses platform-resolved numeric values, including OpenShift namespace ranges. - VM uses the configured numeric guest identity. -UID/GID zero and `u32::MAX` are invalid. Agent commands run as the resolved -non-root user. For Podman's managed `/sandbox` workspace, trusted setup briefly -starts as root to prepare the workspace, then switches to that user before -reading bootstrap material or accepting commands. Identity-changing policy -updates require sandbox recreation, while other policy updates remain live. - -Docker uses an absolute OCI working directory as the workspace. Empty, root, -and explicit `/sandbox` values select `/sandbox`; other paths must already -exist without symlink or reserved-mount collisions and must be usable by the -resolved identity. - -Podman reads the image user and working directory from one pinned image. Empty, -`/`, and explicit `/sandbox` values use the managed `/sandbox` workspace. A -custom path must be absolute, normalized, and outside system and OpenShell -reserved paths. Image and driver mounts cannot cover the workspace. Custom -paths use the image's container filesystem directly; the final non-root user -must be able to write the existing directory. The managed `/sandbox` fallback -uses a driver-owned workspace volume. -Kubernetes and VM use `/sandbox`. +UID/GID zero and `u32::MAX` are invalid. The sandbox and every child start with +the resolved identity and zero capability masks; neither process performs an +in-workload UID transition. Identity-changing policy updates require sandbox +recreation, while other policy updates remain live. + +Docker and Podman use an absolute OCI working directory as the workspace. +Empty, root, and explicit `/sandbox` values select `/sandbox`; other paths must +already exist without symlink or reserved-mount collisions and must be usable +by the resolved identity. Podman keeps a custom workspace in the container +filesystem and uses a driver-owned volume only for `/sandbox`. Kubernetes and +VM use `/sandbox`. ### Executable Identity Binding diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index 2cce47c63f..b1a3049882 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -98,18 +98,6 @@ pub fn validate_container_mount_target(target: &str) -> Result<(), String> { Ok(()) } -/// Validate a mount target against shared and driver-specific control paths. -pub fn validate_container_mount_target_with_control_paths( - target: &str, - control_paths: &[&str], -) -> Result<(), String> { - validate_container_mount_target(target)?; - for control_path in control_paths { - validate_mount_control_path(target, control_path)?; - } - Ok(()) -} - /// Resolve an OCI image working directory to the internal workspace root used /// by local container drivers. /// @@ -372,27 +360,6 @@ mod tests { validate_mount_control_path("/custom-other", "/custom/ssh.sock").unwrap(); } - #[test] - fn container_target_checks_shared_and_driver_control_paths() { - let control_paths = &["/.openshell/channel"]; - assert!( - validate_container_mount_target_with_control_paths( - "/etc/openshell/tls/client", - control_paths, - ) - .is_err() - ); - assert!( - validate_container_mount_target_with_control_paths( - "/.openshell/channel/sandbox", - control_paths, - ) - .is_err() - ); - validate_container_mount_target_with_control_paths("/workspace/cache", control_paths) - .unwrap(); - } - #[test] fn workspace_rejects_malformed_runtime_control_paths() { for control_path in [ diff --git a/crates/openshell-core/src/resource_admission.rs b/crates/openshell-core/src/resource_admission.rs index b0229e0142..7d20c405f4 100644 --- a/crates/openshell-core/src/resource_admission.rs +++ b/crates/openshell-core/src/resource_admission.rs @@ -93,7 +93,6 @@ impl std::str::FromStr for DriverAdmissionConfig { /// Reserved driver-owned runtime metadata; caller labels must never override it. pub const CONFIG_USED_LABEL: &str = "openshell.ai/caller-driver-config-used"; pub const IDENTITIES_LABEL: &str = "openshell.ai/resource-admission-identities"; -pub const PRIVATE_IMAGE_VOLUME_TARGETS_LABEL: &str = "openshell.ai/private-image-volume-targets"; pub fn check_config_provenance(allowed: bool, recorded: Option<&str>) -> Result<(), tonic::Status> { match recorded { diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 831acd3f3f..1f4c91f5e4 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -1215,7 +1215,7 @@ impl DockerComputeDriver { .ok_or_else(|| Status::failed_precondition("sandbox lacks resource identity record"))?; let mut actual = std::collections::BTreeMap::new(); let anonymous_targets: Vec = labels - .get(openshell_core::resource_admission::PRIVATE_IMAGE_VOLUME_TARGETS_LABEL) + .get("openshell.ai/private-image-volume-targets") .and_then(|value| serde_json::from_str(value).ok()) .unwrap_or_default(); for mount in container.mounts.as_deref().unwrap_or_default() { @@ -5657,11 +5657,7 @@ fn build_container_create_body_for_image( driver_mounts::validate_workspace_control_path(&workspace_root, BOUNDARY_MOUNT_PATH) .map_err(Status::failed_precondition)?; for volume in &image.volumes { - driver_mounts::validate_container_mount_target_with_control_paths( - volume, - &[BOUNDARY_MOUNT_PATH], - ) - .map_err(|error| { + driver_mounts::validate_container_mount_target(volume).map_err(|error| { Status::failed_precondition(format!( "invalid image-declared volume '{volume}': {error}" )) @@ -5671,6 +5667,8 @@ fn build_container_create_body_for_image( "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" )) })?; + driver_mounts::validate_mount_control_path(volume, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; } for mount in &driver_config.mounts { let target = match mount { @@ -5681,11 +5679,8 @@ fn build_container_create_body_for_image( }; driver_mounts::validate_workspace_mount_target(target, &workspace_root) .map_err(Status::failed_precondition)?; - driver_mounts::validate_container_mount_target_with_control_paths( - target, - &[BOUNDARY_MOUNT_PATH], - ) - .map_err(Status::failed_precondition)?; + driver_mounts::validate_mount_control_path(target, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; } let mut user_mounts = docker_driver_mounts(driver_config)?; user_mounts.push(Mount { @@ -5709,7 +5704,7 @@ fn build_container_create_body_for_image( }); let mut labels = template.labels.clone(); labels.insert( - openshell_core::resource_admission::PRIVATE_IMAGE_VOLUME_TARGETS_LABEL.into(), + "openshell.ai/private-image-volume-targets".into(), serde_json::to_string(&image.volumes) .map_err(|error| Status::internal(error.to_string()))?, ); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 4596377f26..c53d7d9ffb 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -37,12 +37,13 @@ The supervisor joins the workload's **user namespace only** to preserve UID/GID mapping for shared-volume access. PID, mount, and network namespaces remain separate. The channel volume uses shared SELinux relabeling (`:z`). -Before starting either container, the driver uploads bootstrap files to the -channel volume. For managed `/sandbox`, it also prepares the workspace volume; -custom image workspaces need no upload. Restart restores only the channel -bootstrap, preserving the workspace. The workload starts before the supervisor -so its user namespace exists when the supervisor joins it; a stopped supervisor -resolves that namespace again on its next start. +Before starting either container, the driver uploads volume-relative archives +directly to the channel and workspace volume destinations. A rootfs upload on a +stopped Podman container does not populate nested named volumes. Restart restores +only the channel bootstrap into the existing channel volume, preserving the +workspace. The workload starts before the supervisor so its user namespace exists +when the supervisor joins it; a stopped supervisor resolves that namespace again +on its next start. Custom image workspaces have no workspace volume or upload. The runtime must pass the sandbox's unprivileged enforcement probe, including nested seccomp notification and Landlock. Unsupported runtime defaults fail @@ -92,22 +93,16 @@ environment belong to agent children, never the supervisor process. ## OCI working directory OpenShell reads `WORKDIR` from the workload image. If it is unset, `/`, or -`/sandbox`, OpenShell uses its managed `/sandbox` workspace. A custom path must -be absolute, with no `.` or `..` segments. It cannot overlap `/proc`, `/sys`, +`/sandbox`, OpenShell uses its managed `/sandbox` workspace volume. A custom +path must be absolute and normalized, and cannot overlap `/proc`, `/sys`, `/dev`, OpenShell-reserved paths, or the workload's private control and CA -mounts. For a custom path, image volumes and driver mounts cannot cover the -workspace or one of its parents; mounts nested below it remain valid. Podman -creates a private volume for each image-declared path nested below it. +mounts. Image volumes and driver mounts cannot cover it; mounts nested below it +remain valid. -For a custom path, Podman uses the image's container filesystem directly. -OpenShell keeps the image directory's ownership and permissions, starts as the -final non-root user, and rejects the image if that user cannot reach and write -the directory. Agent commands use the path as their working directory; -`filesystem.include_workdir` grants access to it when enabled. - -For `/sandbox`, OpenShell creates a workspace volume and prepares it before -switching to the non-root user. The separate supervisor receives the path but -does not mount the workspace. +A custom path stays in the image's container filesystem with its ownership and +permissions. The workload starts as the final non-root user, which must be able +to reach and write the directory. Agent commands use the path as their working +directory. ## Lifecycle and readiness @@ -135,8 +130,8 @@ User `bind`, `volume`, `tmpfs`, and `image` mounts and CDI GPU selection remain native Podman features and apply only to the workload. Bind mounts require the operator's `enable_bind_mounts` opt-in and disabled label admission. Supplemental image mounts also require disabled admission. Driver JSON requires -`allow_driver_config = true`. The workload's private mounts cannot be -replaced. User-owned volumes are never created or deleted. +`allow_driver_config = true`. Reserved control paths and the workspace +root cannot be replaced. User-owned volumes are never created or deleted. See [gateway configuration](../../docs/how-it-works/gateways/configuration.mdx) for operator settings and [NETWORKING.md](NETWORKING.md) for supervisor networking. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index f87c7362ef..75cccda548 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -229,7 +229,6 @@ pub struct ResolvedPodmanImage { pub(crate) oci_user: String, pub(crate) environment: Vec, pub(crate) workspace_root: String, - image_volume_targets: Vec, } impl ResolvedPodmanImage { @@ -247,16 +246,11 @@ impl ResolvedPodmanImage { driver_mounts::validate_workspace_control_path(&workspace_root, control_path) .map_err(ComputeDriverError::Precondition)?; } - let mut image_volume_targets = Vec::new(); if workspace_root != driver_mounts::DEFAULT_WORKSPACE_ROOT && let Some(volumes) = image_config.and_then(|config| config.volumes.as_ref()) { for volume in volumes.keys() { - driver_mounts::validate_container_mount_target_with_control_paths( - volume, - PODMAN_WORKLOAD_CONTROL_PATHS, - ) - .map_err(|error| { + validate_podman_mount_target(volume).map_err(|error| { ComputeDriverError::Precondition(format!( "invalid image-declared volume '{volume}': {error}" )) @@ -268,9 +262,7 @@ impl ResolvedPodmanImage { )) }, )?; - image_volume_targets.push(volume.clone()); } - image_volume_targets.sort(); } Ok(Self { id: inspected.id.clone(), @@ -279,10 +271,20 @@ impl ResolvedPodmanImage { .to_string(), environment: image_config.map_or_else(Vec::new, |config| config.env.clone()), workspace_root, - image_volume_targets, }) } + /// Image reference without inspected metadata; uses the managed workspace. + #[cfg(test)] + fn unpinned(image: &str) -> Self { + Self { + id: image.to_string(), + oci_user: String::new(), + environment: Vec::new(), + workspace_root: driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), + } + } + pub(crate) fn uses_managed_workspace(&self) -> bool { self.workspace_root == driver_mounts::DEFAULT_WORKSPACE_ROOT } @@ -301,11 +303,15 @@ pub struct ContainerSpec { volumes: Vec, image_volumes: Vec, hostname: String, - /// Start trusted runtime binaries independently of the image-selected - /// workspace. The resolved workspace is applied to untrusted children. + /// Start trusted runtime binaries from `/` instead of the image + /// `WORKDIR`, so Podman neither creates a missing workdir nor fails to + /// `chdir` before `OpenShell` validates it. Children use the resolved + /// workspace. work_dir: String, /// Overrides the image's ENTRYPOINT. In Podman's libpod API, `command` - /// only overrides CMD (appended as args to the entrypoint). + /// only overrides CMD (appended as args to the entrypoint). We must set + /// `entrypoint` explicitly so the supervisor binary runs directly, + /// regardless of what ENTRYPOINT the sandbox image defines. entrypoint: Vec, command: Vec, user: String, @@ -876,10 +882,7 @@ fn podman_user_mounts( None => {} } driver_mounts::validate_absolute_mount_source(&source, "bind source")?; - driver_mounts::validate_container_mount_target_with_control_paths( - &target, - PODMAN_WORKLOAD_CONTROL_PATHS, - )?; + driver_mounts::validate_container_mount_target(&target)?; result.mounts.push(Mount { kind: "bind".into(), source, @@ -895,10 +898,7 @@ fn podman_user_mounts( } => { reject_subpath(subpath.as_deref(), "podman volume mounts")?; driver_mounts::validate_mount_source(&source, "volume source")?; - driver_mounts::validate_container_mount_target_with_control_paths( - &target, - PODMAN_WORKLOAD_CONTROL_PATHS, - )?; + driver_mounts::validate_container_mount_target(&target)?; result.volumes.push(NamedVolume { name: source, dest: target, @@ -924,10 +924,7 @@ fn podman_user_mounts( { options.push(format!("mode={mode:o}")); } - driver_mounts::validate_container_mount_target_with_control_paths( - &target, - PODMAN_WORKLOAD_CONTROL_PATHS, - )?; + driver_mounts::validate_container_mount_target(&target)?; result.mounts.push(Mount { kind: "tmpfs".into(), source: "tmpfs".into(), @@ -943,10 +940,7 @@ fn podman_user_mounts( } => { reject_subpath(subpath.as_deref(), "podman image mounts")?; driver_mounts::validate_mount_source(&source, "image source")?; - driver_mounts::validate_container_mount_target_with_control_paths( - &target, - PODMAN_WORKLOAD_CONTROL_PATHS, - )?; + driver_mounts::validate_container_mount_target(&target)?; result.image_volumes.push(ImageVolume { source, destination: target, @@ -972,6 +966,14 @@ fn podman_driver_config( Ok(config) } +fn validate_podman_mount_target(target: &str) -> Result<(), String> { + driver_mounts::validate_container_mount_target(target)?; + for control_path in PODMAN_WORKLOAD_CONTROL_PATHS { + driver_mounts::validate_mount_control_path(target, control_path)?; + } + Ok(()) +} + fn validate_podman_driver_mounts( mounts: &[PodmanDriverMountConfig], enable_bind_mounts: bool, @@ -1021,10 +1023,7 @@ fn validate_podman_driver_mounts( target } }; - driver_mounts::validate_container_mount_target_with_control_paths( - target, - PODMAN_WORKLOAD_CONTROL_PATHS, - )?; + validate_podman_mount_target(target)?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(format!( @@ -1148,10 +1147,7 @@ pub fn build_container_spec_with_token_and_gpu_devices( gpu_device_ids: Option<&[String]>, ) -> Result { let image = resolve_image(sandbox, config); - let resolved_image = ResolvedPodmanImage::from_inspect(&ImageInspect { - id: image.to_string(), - config: None, - })?; + let resolved_image = ResolvedPodmanImage::unpinned(image); build_container_spec_for_image( sandbox, config, @@ -1214,23 +1210,10 @@ fn build_base_spec( let resource_limits = build_resource_limits(sandbox, config); let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) .map_err(ComputeDriverError::InvalidArgument)?; - for target in user_mounts - .mounts - .iter() - .map(|mount| mount.destination.as_str()) - .chain( - user_mounts - .volumes - .iter() - .map(|volume| volume.dest.as_str()), - ) - .chain( - user_mounts - .image_volumes - .iter() - .map(|volume| volume.destination.as_str()), - ) - { + let mount_targets = user_mounts.mounts.iter().map(|m| &m.destination); + let volume_targets = user_mounts.volumes.iter().map(|v| &v.dest); + let image_targets = user_mounts.image_volumes.iter().map(|v| &v.destination); + for target in mount_targets.chain(volume_targets).chain(image_targets) { driver_mounts::validate_workspace_mount_target(target, &image.workspace_root) .map_err(ComputeDriverError::Precondition)?; } @@ -1595,11 +1578,6 @@ pub fn build_isolation_specs( workload .labels .insert(crate::isolation::LABEL_ROLE.into(), "sandbox".into()); - workload.labels.insert( - openshell_core::resource_admission::PRIVATE_IMAGE_VOLUME_TARGETS_LABEL.into(), - serde_json::to_string(&input.image.image_volume_targets) - .map_err(|error| ComputeDriverError::Message(error.to_string()))?, - ); workload.env = BTreeMap::new(); workload.unsetenv = input .image @@ -2032,10 +2010,7 @@ mod tests { driver_mounts::DEFAULT_WORKSPACE_ROOT, ] ); - let mut custom_image = resolved_image("sha256:image", "1000:1001", "/workspace/project"); - custom_image - .image_volume_targets - .push("/workspace/project/cache".into()); + let custom_image = resolved_image("sha256:image", "1000:1001", "/workspace/project"); let custom_specs = build_isolation_specs(IsolationSpecInput { sandbox: &sandbox, config: &config, @@ -2051,14 +2026,6 @@ mod tests { }) .unwrap(); assert_eq!(custom_specs.workload.user, "1000:1001"); - assert_eq!( - custom_specs - .workload - .labels - .get(openshell_core::resource_admission::PRIVATE_IMAGE_VOLUME_TARGETS_LABEL) - .map(String::as_str), - Some("[\"/workspace/project/cache\"]") - ); assert!(custom_specs.workload.cap_add.is_empty()); assert_eq!( custom_specs.workload.command, @@ -2160,13 +2127,11 @@ mod tests { let image = ResolvedPodmanImage::from_inspect(&inspect("/workspace/project/cache")) .expect("image volumes nested below the workspace remain valid"); assert_eq!(image.workspace_root, "/workspace/project"); - assert_eq!(image.image_volume_targets, vec!["/workspace/project/cache"]); let mut fallback = inspect("/etc/openshell"); fallback.config.as_mut().unwrap().working_dir = "/sandbox".into(); - let fallback = ResolvedPodmanImage::from_inspect(&fallback) + ResolvedPodmanImage::from_inspect(&fallback) .expect("existing /sandbox images keep their image-volume behavior"); - assert!(fallback.image_volume_targets.is_empty()); } fn json_struct(value: Value) -> prost_types::Struct { @@ -3509,7 +3474,7 @@ mod tests { "mounts": [{ "type": "volume", "source": "work-nfs", - "target": "/opt/openshell/bin" + "target": "/etc/openshell/tls/client" }] }))), ..Default::default() @@ -3521,19 +3486,6 @@ mod tests { let err = try_build_container_spec_with_token(&sandbox, &config, None).unwrap_err(); assert!(err.to_string().contains("reserved OpenShell path")); - - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{"type": "volume", "source": "work-nfs", "target": "/etc/openshell/tls/client"}] - }))); - let err = try_build_container_spec_with_token(&sandbox, &config, None).unwrap_err(); - assert!(err.to_string().contains("reserved OpenShell path")); } #[test] diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 130b17cb17..7dfd0c41ac 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -761,10 +761,6 @@ impl PodmanComputeDriver { .get(openshell_core::resource_admission::IDENTITIES_LABEL) .and_then(|value| serde_json::from_str(value).ok()) .ok_or_else(missing)?; - let anonymous_targets: Vec = labels - .get(openshell_core::resource_admission::PRIVATE_IMAGE_VOLUME_TARGETS_LABEL) - .and_then(|value| serde_json::from_str(value).ok()) - .unwrap_or_default(); let mut actual = std::collections::BTreeMap::new(); for mount in mounts { match mount["Type"].as_str() { @@ -793,16 +789,6 @@ impl PodmanComputeDriver { if !owned || volume.driver != "local" || !volume.options.is_empty() { return Err(missing()); } - } else if !expected.contains_key(name) - && mount["Destination"].as_str().is_some_and(|destination| { - anonymous_targets.iter().any(|target| target == destination) - }) - { - if volume.driver != "local" || !volume.options.is_empty() { - return Err(ComputeDriverError::Precondition( - "image-private volume backing changed".into(), - )); - } } else { actual.insert(name.to_string(), volume.admission_identity()); self.config @@ -1236,11 +1222,7 @@ impl PodmanComputeDriver { .await?; if managed_workspace { self.client - .copy_to_container( - &workload_id, - &resolved_image.workspace_root, - archives.workspace, - ) + .copy_to_container(&workload_id, "/sandbox", archives.workspace) .await?; } let supervisor_id = self @@ -3115,55 +3097,6 @@ mod tests { } } - #[tokio::test] - async fn admission_accepts_image_declared_volume_below_custom_workdir() { - let (socket, requests, handle) = spawn_podman_stub( - "image-volume-admission", - vec![ - StubResponse::new( - StatusCode::OK, - serde_json::json!({ - "Id": "workload-1", - "Name": "workload-1", - "State": {"Status": "created", "Running": false}, - "Config": {"Labels": { - "openshell.ai/sandbox-workspace": "team-a", - "openshell.ai/sandbox-id": "sandbox-1", - "openshell.ai/caller-driver-config-used": "false", - "openshell.ai/resource-admission-identities": "{}", - "openshell.ai/private-image-volume-targets": "[\"/home/app/project/cache\"]" - }}, - "Mounts": [{ - "Type": "volume", - "Name": "anonymous-1", - "Destination": "/home/app/project/cache" - }] - }) - .to_string(), - ), - StubResponse::new( - StatusCode::OK, - serde_json::json!({ - "Name": "anonymous-1", "Driver": "local", "Options": {}, "Labels": {} - }) - .to_string(), - ), - ], - ); - let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - socket_path: Some(socket.clone()), - ..Default::default() - }); - - driver - .admit_container_resources("workload-1") - .await - .expect("a local image-declared volume is private to the workload"); - handle.await.unwrap(); - assert_eq!(requests.lock().unwrap().len(), 2); - let _ = fs::remove_file(socket); - } - #[tokio::test] async fn admission_driver_config_denial_does_not_contact_podman() { for enabled in [true, false] { diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index e03054d651..97948c2d0c 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -267,12 +267,4 @@ Set `process.run_as_user` and `process.run_as_group` in the sandbox policy to ch | Kubernetes | OpenShift SCC namespace annotations, otherwise `1000`. Override with `sandbox_uid` and `sandbox_gid`. | | MicroVM | The image's `sandbox` account, otherwise `1000`. Override with `sandbox_uid` and `sandbox_gid`. | -On Docker, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR`, `/`, or `/sandbox` use `/sandbox`. Any other `WORKDIR` must exist in the image and be writable by the sandbox user. - -Podman also uses the image's `WORKDIR` as the workspace, falling back to -`/sandbox` for an empty, `/`, or `/sandbox` value. A custom path must be an -absolute, normalized path outside system and OpenShell-reserved paths. -Image volumes and driver mounts cannot cover it. Podman uses the image's -container filesystem directly and requires the final non-root user to be able -to write the existing directory. The managed `/sandbox` fallback uses a -workspace volume. Kubernetes and MicroVM continue to use `/sandbox`. +On Docker and Podman, the image's `WORKDIR` becomes the workspace. Images with no `WORKDIR`, `/`, or `/sandbox` use `/sandbox`. Any other `WORKDIR` must exist in the image and be writable by the sandbox user. Kubernetes and MicroVM always use `/sandbox`. diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index 78a57a4a81..6d29b257c3 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -56,12 +56,11 @@ impl ImageGuard { format!( r"FROM {BASE_IMAGE} USER 0:0 -RUN mkdir -p /home/app/project/cache && \ - chown {OCI_UID}:{OCI_GID} /home/app /home/app/project /home/app/project/cache && \ - chmod 0700 /home/app /home/app/project /home/app/project/cache +RUN mkdir -p /home/app/project && \ + chown {OCI_UID}:{OCI_GID} /home/app /home/app/project && \ + chmod 0700 /home/app /home/app/project WORKDIR /home/app/project -RUN printf root-owned > root-owned.txt && chown {OCI_UID}:{OCI_GID} . -VOLUME /home/app/project/cache +RUN printf root-owned > root-owned.txt USER {OCI_UID}:{OCI_GID} " ), @@ -99,6 +98,7 @@ USER {OCI_UID}:{OCI_GID} "Podman-built image has OCI user '{user}', expected {OCI_UID}:{OCI_GID}" )); } + Ok(Self { engine, tag, id }) } } @@ -206,7 +206,6 @@ async fn podman_uses_oci_identity_image_workdir_and_inspected_image_id() { test \"$(pwd -P)\" = /home/app/project; \ test \"$(cat root-owned.txt)\" = root-owned; \ touch direct-workspace-write; \ - touch cache/from-create; \ printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; \ echo podman-oci-identity-ready; sleep infinity", ], @@ -229,9 +228,7 @@ async fn podman_uses_oci_identity_image_workdir_and_inspected_image_id() { test \"$(id -u):$(id -g)\" = 2345:2346; \ test \"$(pwd -P)\" = /home/app/project; \ test -f direct-workspace-write; \ - test -f cache/from-create; \ touch ssh-workspace-write; \ - touch cache/from-ssh; \ echo podman-ssh-identity-ok", ]) .await @@ -317,7 +314,6 @@ async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, contai assert!(!mounts.contains("/etc/openshell/tls")); assert!(!mounts.contains("/.openshell/supervisor")); assert!(!mounts.lines().any(|path| path == "/home/app/project")); - assert!(mounts.lines().any(|path| path == "/home/app/project/cache")); assert!(!mounts.lines().any(|path| path == "/sandbox")); let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/runtime-descriptor.json"]).await.expect("workload cannot read either control credential set"); assert!(posture.contains("0000000000000000")); diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 0d614d90d3..2b83b4b09d 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -280,9 +280,8 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default image workdir. -- Docker also rejects an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. -- A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the reserved paths named in the error. -- Podman also rejects an image volume or driver mount that covers a custom workdir. A custom workdir uses the image's container filesystem directly, without a workspace volume; ensure the sandbox user can write the existing directory. Only the `/sandbox` fallback uses a workspace volume. +- Docker and Podman also reject an image `VOLUME` or driver mount that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. +- A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index 4f729f1133..818d12a269 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -678,12 +678,6 @@ Explicit numeric fields may use any UID/GID from `1` through Warn users that low IDs can inherit permissions from matching accounts, image files, mounted volumes, or devices. -Podman gateways use a normalized absolute OCI `WORKDIR` as the workspace. -Empty, `/`, and explicit `/sandbox` declarations use the managed `/sandbox` -fallback. For a custom path, the final identity must be able to traverse and -write the existing directory in the container filesystem. A custom path does -not use a workspace volume; only the managed `/sandbox` fallback does. - ### Forward ports ```bash From 0f3bdfb1c6700e48499786d0b4231708b37473b8 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 29 Sep 2026 14:05:22 -0700 Subject: [PATCH 19/20] test(oci): share OCI image checks across Docker and Podman Add an oci-image feature testsuite that runs against installed artifacts on Docker rootful, Podman rootful, and Podman rootless tmachine guests. It covers custom WORKDIR placement, image content and ownership, workspace writes from the main process and exec, file transfer, OCI user identity, the managed /sandbox fallback, and rejection of an unwritable WORKDIR. Replace the Docker-only custom_image e2e with the shared suite and keep podman_oci_identity focused on Podman image-ID pinning and identity. Signed-off-by: Matthew Grossman --- .github/workflows/branch-e2e.yml | 5 +- .github/workflows/release-dev.yml | 5 +- .github/workflows/release-tag.yml | 5 +- TESTING.md | 19 +- e2e/rust/Cargo.toml | 5 - e2e/rust/tests/custom_image.rs | 253 ------------- e2e/rust/tests/podman_oci_identity.rs | 29 +- .../ansible/playbooks/features/oci-image.yaml | 117 ++++++ tests/artifacts.nix | 11 + tests/config.nix | 7 + tests/suites/features/Cargo.lock | 9 + tests/suites/features/Cargo.toml | 2 +- tests/suites/features/oci-image/Cargo.toml | 12 + .../features/oci-image/tests/oci_image.rs | 346 ++++++++++++++++++ 14 files changed, 543 insertions(+), 282 deletions(-) delete mode 100644 e2e/rust/tests/custom_image.rs create mode 100644 tests/ansible/playbooks/features/oci-image.yaml create mode 100644 tests/suites/features/oci-image/Cargo.toml create mode 100644 tests/suites/features/oci-image/tests/oci_image.rs diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 4b939f79d2..f3a4f4e04d 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -243,7 +243,10 @@ jobs: test-matrix: >- [ {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"provider-refresh"}, - {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"provider-refresh"} + {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"provider-refresh"}, + {"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"oci-image"} ] # Run driver-specific integration tests: diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 371144bf7c..99a8506ca7 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -137,7 +137,10 @@ jobs: test-matrix: >- [ {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"provider-refresh"}, - {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"provider-refresh"} + {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"provider-refresh"}, + {"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"oci-image"} ] docker-e2e: diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 59c2a542b4..aa61f46460 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -187,7 +187,10 @@ jobs: test-matrix: >- [ {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"provider-refresh"}, - {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"provider-refresh"} + {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"provider-refresh"}, + {"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"oci-image"} ] docker-e2e: diff --git a/TESTING.md b/TESTING.md index 59031f242f..46218510fb 100644 --- a/TESTING.md +++ b/TESTING.md @@ -279,6 +279,23 @@ binary. `tests/artifacts.nix` keeps the follow-up exclusions explicit and uses the same filter for the generated inventory, so excluded binaries cannot appear as false passes or silently re-enter the archive. +The `oci-image` feature testsuite (`tests/suites/features/oci-image`) checks +OCI image identity and working-directory behavior shared by the Docker and +Podman drivers against installed artifacts. CI runs it on Docker rootful, +Podman rootful, and Podman rootless guests: + +```shell +nix run .#tmachine -- test ubuntu-docker-rootful binaries oci-image +``` + +Run it against a local gateway by naming the command that builds images into +the gateway's image store: + +```shell +OPENSHELL_TEST_CONTAINER_ENGINE=podman e2e/with-podman-gateway.sh \ + cargo test --manifest-path tests/suites/features/Cargo.toml -p openshell-test-feature-oci-image -- --test-threads 1 +``` + Run the VM-backed Rust CLI e2e suite: ```shell @@ -458,7 +475,7 @@ cargo test --manifest-path e2e/rust/Cargo.toml --features e2e --test sync Run a single Docker-only test directly with cargo: ```shell -cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test custom_image +cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test docker_preflight ``` The harness (`e2e/rust/src/harness/`) provides: diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index b492c8ac86..7641a5872e 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -53,11 +53,6 @@ name = "vm_overlay" path = "tests/vm_overlay.rs" required-features = ["e2e-vm"] -[[test]] -name = "custom_image" -path = "tests/custom_image.rs" -required-features = ["e2e-docker"] - [[test]] name = "rootfs_tar" path = "tests/rootfs_tar.rs" diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs deleted file mode 100644 index 96fdd56612..0000000000 --- a/e2e/rust/tests/custom_image.rs +++ /dev/null @@ -1,253 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-local-container-driver")] - -//! E2E test: build custom container images and run sandboxes with them. -//! -//! Prerequisites: -//! - A running Docker- or Podman-backed openshell gateway -//! - The matching container runtime running (for image builds) -//! - The `openshell` binary (built automatically from the workspace) - -use std::{fs, io::Write}; - -use openshell_e2e::harness::container::ImageGuard; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use serial_test::serial; - -const DOCKERFILE_CONTENT: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim - -# iproute2 is required for sandbox network namespace isolation. -RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ - && rm -rf /var/lib/apt/lists/* - -RUN groupadd -g 1235 appstaff && \ - useradd -m -u 1234 -g appstaff app - -# The final image identity already owns the OCI working directory. Existing -# root-owned content remains root-owned. -WORKDIR /workspace/project -RUN printf root-owned > root-owned.txt && chown app:appstaff . - -# Write a marker file so we can verify this is our custom image. -# Place under /etc (Landlock baseline read-only path) so the sandbox -# can read it when filesystem restrictions are properly enforced. -RUN echo "custom-image-e2e-marker" > /etc/marker.txt - -USER app -CMD ["sleep", "infinity"] -"#; - -const NUMERIC_DOCKERFILE_CONTENT: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim - -RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ - && rm -rf /var/lib/apt/lists/* - -USER 2345:2346 -CMD ["sleep", "infinity"] -"#; - -const UNWRITABLE_WORKDIR_DOCKERFILE_CONTENT: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim - -RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ - && rm -rf /var/lib/apt/lists/* \ - && groupadd -g 3235 appstaff \ - && useradd -m -u 3234 -g appstaff app - -WORKDIR /workspace/project -USER app -CMD ["sleep", "infinity"] -"#; - -const MARKER: &str = "custom-image-e2e-marker"; - -/// A named OCI user can write through direct and SSH children when the image -/// already grants that authority; existing content retains its ownership. -#[tokio::test] -#[serial(custom_image)] -async fn sandbox_from_custom_image() { - // Step 1: Write a temporary Dockerfile. - let tmpdir = tempfile::tempdir().expect("create tmpdir"); - let dockerfile_path = tmpdir.path().join("Dockerfile"); - { - let mut f = std::fs::File::create(&dockerfile_path).expect("create Dockerfile"); - f.write_all(DOCKERFILE_CONTENT.as_bytes()) - .expect("write Dockerfile"); - } - - // Step 2: Build the image out-of-band and create a sandbox from it. - // `--from` no longer builds local Dockerfiles itself (pre-0.1.0 - // breaking change); tests build explicitly and pass the resulting tag. - let image = ImageGuard::build("custom-dockerfile", &dockerfile_path, tmpdir.path()) - .expect("build custom image with selected container engine"); - let mut guard = SandboxGuard::create_keep_with_args( - &["--from", image.tag(), "--no-tty"], - &[ - "sh", - "-c", - "set -eu; id -u; id -g; test \"$(pwd -P)\" = /workspace/project; \ - test \"$HOME\" = /workspace/project; test \"$(cat root-owned.txt)\" = root-owned; \ - test \"$(stat -c %u:%g .)\" = 1234:1235; \ - test \"$(stat -c %u:%g root-owned.txt)\" = 0:0; \ - touch direct-oci-user-write; cat /etc/marker.txt; echo Ready; sleep infinity", - ], - "Ready", - ) - .await - .expect("sandbox create from custom image"); - - // Step 3: Verify the marker file content appears in the output. - let clean_output = strip_ansi(&guard.create_output); - assert!( - clean_output.contains(MARKER), - "expected marker '{MARKER}' in sandbox output:\n{clean_output}" - ); - assert!( - clean_output.contains("1234") && clean_output.contains("1235"), - "expected named OCI identity 1234:1235 in sandbox output:\n{clean_output}" - ); - - let ssh_output = guard - .exec(&[ - "sh", - "-c", - "set -eu; test \"$(id -u):$(id -g)\" = 1234:1235; \ - test \"$(pwd -P)\" = /workspace/project; test \"$HOME\" = /workspace/project; \ - touch ssh-oci-user-write; echo ssh-write-ok", - ]) - .await - .expect("SSH child should write to prepared workspace"); - assert!( - ssh_output.contains("ssh-write-ok"), - "expected SSH write marker:\n{ssh_output}" - ); - - let transfer_source = tmpdir.path().join("workspace-transfer.txt"); - fs::write(&transfer_source, "workspace-transfer-ok").expect("write transfer fixture"); - guard - .upload_to_workdir( - transfer_source - .to_str() - .expect("transfer fixture path is UTF-8"), - ) - .await - .expect("upload should default to the OCI workspace"); - let transfer_download = tmpdir.path().join("workspace-transfer-downloaded.txt"); - guard - .download( - "workspace-transfer.txt", - transfer_download - .to_str() - .expect("download destination path is UTF-8"), - ) - .await - .expect("download should resolve relative to the OCI workspace"); - assert_eq!( - fs::read_to_string(transfer_download).expect("read downloaded transfer fixture"), - "workspace-transfer-ok" - ); - - guard - .exec(&[ - "sh", - "-c", - "set -eu; mkdir -p merge-upload; \ - printf remote-conflict > merge-upload/conflict.txt; \ - printf remote-preserved > merge-upload/unrelated.txt", - ]) - .await - .expect("seed existing remote upload directory"); - let merge_source = tmpdir.path().join("merge-upload"); - fs::create_dir(&merge_source).expect("create local upload directory"); - fs::write(merge_source.join("conflict.txt"), "local-conflict") - .expect("write conflicting local upload file"); - fs::write(merge_source.join("added.txt"), "local-added") - .expect("write added local upload file"); - guard - .upload_to_workdir(merge_source.to_str().expect("merge upload path is UTF-8")) - .await - .expect("upload should merge into the existing remote directory"); - guard - .exec(&[ - "sh", - "-c", - "set -eu; \ - test \"$(cat merge-upload/conflict.txt)\" = local-conflict; \ - test \"$(cat merge-upload/added.txt)\" = local-added; \ - test \"$(cat merge-upload/unrelated.txt)\" = remote-preserved", - ]) - .await - .expect("upload should overwrite conflicts and preserve unrelated remote files"); - - // Explicit cleanup (also happens in Drop, but explicit is clearer in tests). - guard.cleanup().await; -} - -/// A numeric OCI user/group pair works without passwd or group entries. -/// The image intentionally has no pre-existing `/sandbox`. -#[tokio::test] -#[serial(custom_image)] -async fn sandbox_from_passwd_less_numeric_oci_user() { - let tmpdir = tempfile::tempdir().expect("create tmpdir"); - let dockerfile_path = tmpdir.path().join("Dockerfile"); - { - let mut f = std::fs::File::create(&dockerfile_path).expect("create Dockerfile"); - f.write_all(NUMERIC_DOCKERFILE_CONTENT.as_bytes()) - .expect("write Dockerfile"); - } - - let image = ImageGuard::build("passwd-less-numeric", &dockerfile_path, tmpdir.path()) - .expect("build numeric OCI image with selected container engine"); - let mut guard = SandboxGuard::create(&[ - "--from", - image.tag(), - "--", - "sh", - "-c", - "set -eu; id -u; id -g; test \"$(pwd -P)\" = /sandbox; \ - test \"$HOME\" = /sandbox; touch numeric-oci-user-write", - ]) - .await - .expect("sandbox create from numeric OCI Dockerfile"); - - let clean_output = strip_ansi(&guard.create_output); - assert!( - clean_output.contains("2345") && clean_output.contains("2346"), - "expected numeric OCI identity 2345:2346 in sandbox output:\n{clean_output}" - ); - - guard.cleanup().await; -} - -#[tokio::test] -#[serial(custom_image)] -async fn sandbox_rejects_image_workdir_that_would_require_new_authority() { - let tmpdir = tempfile::tempdir().expect("create tmpdir"); - let dockerfile_path = tmpdir.path().join("Dockerfile"); - fs::write(&dockerfile_path, UNWRITABLE_WORKDIR_DOCKERFILE_CONTENT).expect("write Dockerfile"); - let image = ImageGuard::build("unwritable-workdir", &dockerfile_path, tmpdir.path()) - .expect("build unwritable-workdir image with selected container engine"); - - let result = SandboxGuard::create_keep_with_args( - &["--from", image.tag(), "--no-tty"], - &["sh", "-c", "echo should-not-run"], - "should-not-run", - ) - .await; - let error = match result { - Ok(mut guard) => { - guard.cleanup().await; - panic!("root-owned workdir must not be made writable for the image user"); - } - Err(error) => error, - }; - let message = error.to_string(); - assert!( - (message.contains("WorkspaceValidationFailed") && message.contains("WorkingDir")) - || message.contains("subsystem request failed") - || message.contains("image workspace validation failed"), - "expected rejected image to fail provisioning, got: {message}" - ); -} diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index 6d29b257c3..8820e23406 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -3,13 +3,15 @@ #![cfg(feature = "e2e-podman")] -//! Podman-specific E2E coverage for OCI identity/workspace inspection, -//! direct image-workspace access, and immutable-image launch. +//! Podman-specific E2E coverage for OCI identity inspection and immutable-image +//! launch. //! //! The test builds an image through the selected Podman engine, creates a -//! sandbox from its mutable tag, and verifies the child identity, image -//! content, workspace placement, and image ID recorded on the real sandbox -//! container. +//! sandbox from its mutable tag, and verifies both the child identity and the +//! image ID recorded on the real sandbox container. This exercises the Podman +//! API inspect → protected metadata → create path rather than only its unit +//! serialization boundaries. Workspace behavior shared with Docker is covered +//! by the `oci-image` feature suite in `tests/suites/features`. use std::process::Stdio; @@ -60,7 +62,6 @@ RUN mkdir -p /home/app/project && \ chown {OCI_UID}:{OCI_GID} /home/app /home/app/project && \ chmod 0700 /home/app /home/app/project WORKDIR /home/app/project -RUN printf root-owned > root-owned.txt USER {OCI_UID}:{OCI_GID} " ), @@ -184,7 +185,7 @@ fn normalized_image_id(image_id: &str) -> &str { } #[tokio::test] -async fn podman_uses_oci_identity_image_workdir_and_inspected_image_id() { +async fn podman_uses_oci_identity_and_inspected_image_id() { if !is_e2e_driver("podman") { eprintln!("Skipping Podman OCI identity test: e2e driver is not podman"); return; @@ -202,12 +203,7 @@ async fn podman_uses_oci_identity_image_workdir_and_inspected_image_id() { &[ "sh", "-c", - "set -eu; \ - test \"$(pwd -P)\" = /home/app/project; \ - test \"$(cat root-owned.txt)\" = root-owned; \ - touch direct-workspace-write; \ - printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; \ - echo podman-oci-identity-ready; sleep infinity", + "set -eu; printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; echo podman-oci-identity-ready; sleep infinity", ], READY_MARKER, ) @@ -224,12 +220,7 @@ async fn podman_uses_oci_identity_image_workdir_and_inspected_image_id() { .exec(&[ "sh", "-c", - "set -eu; \ - test \"$(id -u):$(id -g)\" = 2345:2346; \ - test \"$(pwd -P)\" = /home/app/project; \ - test -f direct-workspace-write; \ - touch ssh-workspace-write; \ - echo podman-ssh-identity-ok", + "test \"$(id -u):$(id -g)\" = 2345:2346; echo podman-ssh-identity-ok", ]) .await .expect("SSH child should use Podman OCI identity"); diff --git a/tests/ansible/playbooks/features/oci-image.yaml b/tests/ansible/playbooks/features/oci-image.yaml new file mode 100644 index 0000000000..96cc039898 --- /dev/null +++ b/tests/ansible/playbooks/features/oci-image.yaml @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Run OCI image feature tests + hosts: all + gather_facts: false + vars: + oci_image_test_root: /var/lib/openshell-oci-image/tests + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Detect tmachine container runtime + ansible.builtin.include_role: + name: tmachine_container_runtime + + # Tests build images into the store the gateway reads. Rootful Podman's + # store belongs to root, so the unprivileged test user builds through sudo. + - name: Select the gateway's container engine command + ansible.builtin.set_fact: + oci_image_container_engine: >- + {{ 'docker' if tmachine_container_runtime_name == 'docker' + else 'podman' if tmachine_container_runtime_is_rootless + else 'sudo -n podman' }} + + - name: Create OCI image test directory + become: true + ansible.builtin.file: + path: "{{ oci_image_test_root }}" + state: directory + owner: tmachine + group: tmachine + mode: "0700" + + - name: Extract OCI image test bundle + become: true + ansible.builtin.unarchive: + src: "{{ oci_image_test_bundle }}" + dest: "{{ oci_image_test_root }}" + owner: tmachine + group: tmachine + + - name: Check OCI image nextest archive + ansible.builtin.stat: + path: "{{ oci_image_test_root }}/tests.tar.zst" + register: oci_image_archive + + - name: Require OCI image nextest archive + ansible.builtin.assert: + that: + - oci_image_archive.stat.isreg | default(false) + fail_msg: OCI image test bundle did not contain tests.tar.zst + + - name: Resolve installed OpenShell CLI + ansible.builtin.command: + argv: + - /bin/sh + - -c + - command -v openshell + register: openshell_cli + changed_when: false + + - name: Run OCI image archive + ansible.builtin.command: + argv: + - cargo-nextest + - nextest + - run + - --archive-file + - "{{ oci_image_test_root }}/tests.tar.zst" + - --workspace-remap + - "{{ oci_image_test_root }}" + - --no-capture + # The guest has 4 GiB; keep sandbox creates from competing for it. + - --test-threads + - "1" + - --no-fail-fast + environment: + HOME: /home/tmachine + OPENSHELL_BIN: "{{ openshell_cli.stdout }}" + OPENSHELL_TEST_CONTAINER_ENGINE: "{{ oci_image_container_engine }}" + XDG_RUNTIME_DIR: "/run/user/{{ tmachine_container_runtime_tmachine_uid.stdout }}" + register: oci_image_result + changed_when: false + failed_when: false + + - name: Show OCI image diagnostics + ansible.builtin.debug: + var: oci_image_result + when: oci_image_result.rc != 0 + + - name: Read OpenShell gateway logs + become: true + ansible.builtin.command: + argv: + - journalctl + - --unit + - openshell-gateway.service + - --no-pager + - --lines + - "500" + register: openshell_gateway_logs + changed_when: false + failed_when: false + when: oci_image_result.rc != 0 + + - name: Show OpenShell gateway logs + ansible.builtin.debug: + var: openshell_gateway_logs.stdout_lines + when: oci_image_result.rc != 0 + + - name: Require OCI image success + ansible.builtin.assert: + that: + - oci_image_result.rc == 0 + fail_msg: OCI image feature tests failed diff --git a/tests/artifacts.nix b/tests/artifacts.nix index c3017a5da5..398d0a2408 100644 --- a/tests/artifacts.nix +++ b/tests/artifacts.nix @@ -95,6 +95,14 @@ let target = muslToolchain.target; output = "artifacts/test-archives/${muslToolchain.target}/provider-refresh-keycloak-tests.tar"; }; + ociImageArchive = mkTestArchive { + name = "oci-image"; + workspacePath = "tests/suites/features"; + manifestPath = "tests/suites/features/Cargo.toml"; + package = "openshell-test-feature-oci-image"; + target = muslToolchain.target; + output = "artifacts/test-archives/${muslToolchain.target}/oci-image-tests.tar"; + }; # Follow-up: migrate these wrapper-coupled tests once tmachine provides their # managed-gateway controls, SPIFFE fixtures, caller driver-config setting, @@ -202,6 +210,7 @@ rec { inherit conformanceCliArchive providerRefreshKeycloakArchive + ociImageArchive podmanDriverArchive podmanE2eArchive podmanE2eCiTests @@ -250,12 +259,14 @@ rec { runtimeInputs = [ conformanceCliArchive providerRefreshKeycloakArchive + ociImageArchive podmanDriverArchive podmanE2eArchive ]; text = '' build-openshell-conformance-test-archive build-provider-refresh-keycloak-test-archive + build-oci-image-test-archive build-podman-driver-test-archive build-podman-e2e-test-archive ''; diff --git a/tests/config.nix b/tests/config.nix index a19b81cee8..ee4ee64eef 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -133,6 +133,13 @@ let provider_refresh_keycloak_test_bundle = "../artifacts/test-archives/${muslTarget}/provider-refresh-keycloak-tests.tar"; }; } + { + name = "oci-image"; + playbooks = [ "ansible/playbooks/features/oci-image.yaml" ]; + inputs = { + oci_image_test_bundle = "../artifacts/test-archives/${muslTarget}/oci-image-tests.tar"; + }; + } { name = "e2e-podman"; playbooks = [ "ansible/playbooks/drivers/podman/e2e.yaml" ]; diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 268f8bac92..97b4048d6a 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -916,6 +916,15 @@ dependencies = [ "url", ] +[[package]] +name = "openshell-test-feature-oci-image" +version = "0.0.0" +dependencies = [ + "openshell-conformance", + "tempfile", + "tokio", +] + [[package]] name = "openshell-test-feature-provider-refresh-keycloak" version = "0.0.0" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index 7acd72b5d2..9259450a16 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["provider-refresh/keycloak"] +members = ["oci-image", "provider-refresh/keycloak"] diff --git a/tests/suites/features/oci-image/Cargo.toml b/tests/suites/features/oci-image/Cargo.toml new file mode 100644 index 0000000000..1107587ca7 --- /dev/null +++ b/tests/suites/features/oci-image/Cargo.toml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-test-feature-oci-image" +version = "0.0.0" +edition = "2024" + +[dependencies] +openshell-conformance = { path = "../../../../crates/openshell-conformance" } +tempfile = "3" +tokio = { version = "1.43", features = ["macros", "rt"] } diff --git a/tests/suites/features/oci-image/tests/oci_image.rs b/tests/suites/features/oci-image/tests/oci_image.rs new file mode 100644 index 0000000000..3bd6423252 --- /dev/null +++ b/tests/suites/features/oci-image/tests/oci_image.rs @@ -0,0 +1,346 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! OCI image behavior shared by the Docker and Podman compute drivers. +//! +//! Each test builds a small image with the container engine that backs the +//! target gateway, creates a sandbox from it through the candidate CLI, and +//! checks the process identity, workspace, and image content seen by both the +//! sandbox main process and `sandbox exec`. +//! +//! `OPENSHELL_BIN` names the candidate CLI. `OPENSHELL_TEST_CONTAINER_ENGINE` +//! is the command that builds images into the gateway's image store, such as +//! `docker`, `podman`, or `sudo -n podman`. + +use std::process::Command; +use std::time::Duration; + +use openshell_conformance::OpenShellRunner; + +const BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; +const ENGINE_ENV: &str = "OPENSHELL_TEST_CONTAINER_ENGINE"; +const CREATE_TIMEOUT: Duration = Duration::from_mins(10); +const COMMAND_TIMEOUT: Duration = Duration::from_mins(2); + +/// A named image user that owns its custom `WORKDIR`. Existing image content +/// keeps its ownership. +#[tokio::test] +async fn custom_workdir_with_named_user() { + run("oci-image/custom-workdir-named-user", async |runner| { + let image = TestImage::build( + "named-workdir", + &format!( + "FROM {BASE_IMAGE} +RUN groupadd -g 1235 appstaff && useradd -m -u 1234 -g appstaff app +WORKDIR /workspace/project +RUN printf root-owned > root-owned.txt && chown app:appstaff . +USER app +" + ), + )?; + let checks = format!( + "{} test \"$(stat -c %u:%g .)\" = 1234:1235;", + workspace_checks("1234:1235", "/workspace/project", true) + ); + let sandbox = create_sandbox(runner, "named", "nu", &image, &checks).await?; + file_transfer_uses_workspace(runner, &sandbox).await + }) + .await; +} + +/// A numeric image user without passwd entries can reach a custom `WORKDIR` +/// whose parent directories are private to that user. +#[tokio::test] +async fn custom_workdir_with_numeric_user_and_private_parents() { + run("oci-image/custom-workdir-numeric-user", async |runner| { + let image = TestImage::build( + "numeric-workdir", + &format!( + "FROM {BASE_IMAGE} +RUN mkdir -p /home/app/project && \\ + chown 2345:2346 /home/app /home/app/project && \\ + chmod 0700 /home/app /home/app/project +WORKDIR /home/app/project +RUN printf root-owned > root-owned.txt +USER 2345:2346 +" + ), + )?; + let checks = workspace_checks("2345:2346", "/home/app/project", true); + create_sandbox(runner, "numeric", "pu", &image, &checks) + .await + .map(drop) + }) + .await; +} + +/// An image without a `WORKDIR` uses the managed `/sandbox` workspace. An +/// image that declares `USER` provides a `/sandbox` owned by that user. +#[tokio::test] +async fn default_workdir_uses_managed_workspace() { + run("oci-image/default-workdir", async |runner| { + let image = TestImage::build( + "default-workdir", + &format!( + "FROM {BASE_IMAGE} +RUN mkdir /sandbox && chown 2345:2346 /sandbox && chmod 0700 /sandbox +USER 2345:2346 +" + ), + )?; + let checks = workspace_checks("2345:2346", "/sandbox", false); + create_sandbox(runner, "default", "dw", &image, &checks) + .await + .map(drop) + }) + .await; +} + +/// OpenShell rejects a custom `WORKDIR` that the image user cannot write +/// instead of granting the user new access to it. +#[tokio::test] +async fn unwritable_custom_workdir_is_rejected() { + run("oci-image/unwritable-workdir", async |runner| { + let image = TestImage::build( + "unwritable-workdir", + &format!( + "FROM {BASE_IMAGE} +RUN groupadd -g 3235 appstaff && useradd -m -u 3234 -g appstaff app +WORKDIR /workspace/project +USER app +" + ), + )?; + let name = format!("oi-{}-uw", runner.id()); + runner.track_sandbox(&name); + let create = runner + .step("unwritable/create") + .description("sandbox creation fails before the command runs") + .with_timeout(CREATE_TIMEOUT) + .run(&[ + "sandbox", + "create", + "--name", + &name, + "--from", + &image.tag, + "--no-tty", + "--", + "sh", + "-c", + "echo should-not-run", + ]) + .await + .map_err(|error| error.to_string())?; + if create.success() || create.stdout().contains("should-not-run") { + return Err(create.failure_diagnostic("sandbox creation fails before the command runs")); + } + Ok(()) + }) + .await; +} + +async fn run(scenario: &str, test: impl AsyncFnOnce(&mut OpenShellRunner) -> Result<(), String>) { + let mut runner = + OpenShellRunner::from_env(scenario).expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + test(&mut runner).await + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("{scenario} failed:\n{error}"); + } +} + +/// Shell checks for the identity, working directory, and `HOME` of a sandbox +/// child. With `image_file`, also check that root-owned image content is +/// present and unchanged in the workspace. +fn workspace_checks(identity: &str, workspace: &str, image_file: bool) -> String { + let mut checks = format!( + "test \"$(id -u):$(id -g)\" = {identity}; \ + test \"$(pwd -P)\" = {workspace}; \ + test \"$HOME\" = {workspace};" + ); + if image_file { + checks.push_str( + " test \"$(cat root-owned.txt)\" = root-owned; \ + test \"$(stat -c %u:%g root-owned.txt)\" = 0:0;", + ); + } + checks +} + +/// Create a detached sandbox whose main process runs `checks` and writes to +/// the workspace, then run the same checks and a write through `sandbox exec`. +async fn create_sandbox( + runner: &mut OpenShellRunner, + suffix: &str, + short: &str, + image: &TestImage, + checks: &str, +) -> Result { + // Sandbox names are limited to 19 characters on some drivers. + let name = format!("oi-{}-{short}", runner.id()); + runner.track_sandbox(&name); + let main = format!( + "(set -eu; {checks} touch main-write) >/tmp/oci-main.log 2>&1; \ + echo $? >/tmp/oci-main.status; exec sleep infinity" + ); + runner + .step(format!("{suffix}/create")) + .description("sandbox starts from the test image") + .with_timeout(CREATE_TIMEOUT) + .run(&[ + "sandbox", "create", "--name", &name, "--from", &image.tag, "--detach", "--", "sh", + "-c", &main, + ]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + + let exec = format!( + "set -eu; i=0; \ + while [ ! -f /tmp/oci-main.status ]; do \ + i=$((i + 1)); [ \"$i\" -le 60 ] || {{ echo main process checks did not finish >&2; exit 1; }}; \ + sleep 1; \ + done; \ + if [ \"$(cat /tmp/oci-main.status)\" != 0 ]; then \ + echo main process checks failed: >&2; cat /tmp/oci-main.log >&2; exit 1; \ + fi; \ + test -f main-write; {checks} touch exec-write" + ); + runner + .step(format!("{suffix}/exec")) + .description("main process and exec children see the image workspace") + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", "exec", "--name", &name, "--no-tty", "--", "sh", "-c", &exec, + ]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + Ok(name) +} + +/// Upload and download default to paths relative to the image workspace. +async fn file_transfer_uses_workspace( + runner: &OpenShellRunner, + sandbox: &str, +) -> Result<(), String> { + let local = tempfile::tempdir().map_err(|error| format!("create temp dir: {error}"))?; + let upload = local.path().join("oci-transfer.txt"); + std::fs::write(&upload, "oci-transfer-ok").map_err(|error| format!("write upload: {error}"))?; + let upload = upload.to_str().ok_or("upload path is not UTF-8")?; + runner + .step("named/upload") + .description("upload without a destination writes to the workspace") + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "upload", sandbox, upload, "--no-git-ignore"]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + runner + .step("named/uploaded") + .description("uploaded file is in the workspace") + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + sandbox, + "--no-tty", + "--", + "sh", + "-c", + "test \"$(cat oci-transfer.txt)\" = oci-transfer-ok", + ]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + + let download = local.path().join("downloaded.txt"); + let download_path = download.to_str().ok_or("download path is not UTF-8")?; + runner + .step("named/download") + .description("download resolves relative paths in the workspace") + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "download", + sandbox, + "oci-transfer.txt", + download_path, + ]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + let downloaded = + std::fs::read_to_string(&download).map_err(|error| format!("read download: {error}"))?; + if downloaded != "oci-transfer-ok" { + return Err(format!( + "downloaded file has unexpected content: {downloaded:?}" + )); + } + Ok(()) +} + +/// An image built into the gateway's image store and removed on drop. +struct TestImage { + engine: Vec, + tag: String, +} + +impl TestImage { + fn build(name: &str, containerfile: &str) -> Result { + let engine: Vec = std::env::var(ENGINE_ENV) + .map_err(|_| format!("{ENGINE_ENV} must name the gateway's container engine"))? + .split_whitespace() + .map(str::to_string) + .collect(); + if engine.is_empty() { + return Err(format!("{ENGINE_ENV} is empty")); + } + let context = tempfile::tempdir().map_err(|error| format!("create context: {error}"))?; + let file = context.path().join("Containerfile"); + std::fs::write(&file, containerfile) + .map_err(|error| format!("write Containerfile: {error}"))?; + let image = Self { + engine, + tag: format!("localhost/openshell-test-oci-{name}:{}", std::process::id()), + }; + image.engine_command(&[ + "build", + "--file", + file.to_str().ok_or("Containerfile path is not UTF-8")?, + "--tag", + &image.tag, + context.path().to_str().ok_or("context path is not UTF-8")?, + ])?; + Ok(image) + } + + fn engine_command(&self, args: &[&str]) -> Result<(), String> { + let command = format!("{} {}", self.engine.join(" "), args.join(" ")); + let output = Command::new(&self.engine[0]) + .args(&self.engine[1..]) + .args(args) + .output() + .map_err(|error| format!("failed to run {command}: {error}"))?; + if !output.status.success() { + return Err(format!( + "{command} failed ({}):\n{}{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + Ok(()) + } +} + +impl Drop for TestImage { + fn drop(&mut self) { + let _ = self.engine_command(&["image", "rm", "--force", &self.tag]); + } +} From 6a347f4d9ada484e14a4f494086c9cc7722dbaa4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 29 Sep 2026 16:23:12 -0700 Subject: [PATCH 20/20] fix(podman): create managed workspace volumes owned by the workload identity Podman now creates the managed /sandbox volume with uid/gid options for the resolved workload identity, so the workload starts directly as that identity. This removes the root-then-drop workspace chown and fixes rootful sandboxes whose image USER or policy run_as_user could not write to a root-owned /sandbox. Signed-off-by: Matthew Grossman --- crates/openshell-driver-podman/src/client.rs | 69 +++++++++-- .../openshell-driver-podman/src/container.rs | 110 ++++-------------- crates/openshell-driver-podman/src/driver.rs | 37 ++++-- .../features/oci-image/tests/oci_image.rs | 11 +- 4 files changed, 118 insertions(+), 109 deletions(-) diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 84301f85d1..84b21e79e7 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -705,12 +705,30 @@ impl PodmanClient { // ── Volume operations ──────────────────────────────────────────────── /// Never adopt an unrelated existing volume on a private provisioning path. + /// + /// With `owner`, Podman creates the volume root owned by that UID and GID, + /// so a non-root workload can use it without a privileged chown. pub(crate) async fn create_owned_volume( &self, name: &str, sandbox_id: &str, workspace: &str, + owner: Option<(u32, u32)>, ) -> Result<(), PodmanApiError> { + let mount_options = owner.map(|(uid, gid)| format!("uid={uid},gid={gid}")); + // Podman records the parsed `UID`/`GID` next to the raw `o` option. + let owned_as_requested = |options: &HashMap| { + let Some((uid, gid)) = owner else { + return options.is_empty(); + }; + options.get("o") == mount_options.as_ref() + && options.iter().all(|(key, value)| match key.as_str() { + "o" => true, + "UID" => *value == uid.to_string(), + "GID" => *value == gid.to_string(), + _ => false, + }) + }; let labels = HashMap::from([ ( openshell_core::driver_utils::LABEL_SANDBOX_ID.to_string(), @@ -724,7 +742,7 @@ impl PodmanClient { match self.inspect_volume(name).await { Ok(existing) => { if existing.driver != "local" - || !existing.options.is_empty() + || !owned_as_requested(&existing.options) || existing.labels.as_ref() != Some(&labels) { return Err(PodmanApiError::InvalidInput( @@ -736,14 +754,15 @@ impl PodmanClient { Err(PodmanApiError::NotFound(_)) => {} Err(error) => return Err(error), } - self.create_ignore_conflict( - "/libpod/volumes/create", - &serde_json::json!({"Name":name,"Driver":"local","Labels":labels}), - ) - .await?; + let mut body = serde_json::json!({"Name":name,"Driver":"local","Labels":labels}); + if let Some(mount_options) = &mount_options { + body["Options"] = serde_json::json!({ "o": mount_options }); + } + self.create_ignore_conflict("/libpod/volumes/create", &body) + .await?; let created = self.inspect_volume(name).await?; if created.driver != "local" - || !created.options.is_empty() + || !owned_as_requested(&created.options) || created.labels.as_ref() != Some(&labels) { return Err(PodmanApiError::InvalidInput( @@ -1161,6 +1180,42 @@ mod tests { let _ = std::fs::remove_file(socket_path); } + #[tokio::test] + async fn create_owned_volume_verifies_requested_owner() { + let labels = + r#"{"openshell.ai/sandbox-id":"sandbox-1","openshell.ai/sandbox-workspace":"team-a"}"#; + for (options, accepted) in [ + ( + r#"{"o":"uid=1234,gid=1235","UID":"1234","GID":"1235"}"#, + true, + ), + (r#"{"o":"uid=1234,gid=1235"}"#, true), + (r#"{"o":"uid=1234,gid=1235","UID":"0","GID":"1235"}"#, false), + (r#"{"o":"uid=1234,gid=1235","device":"/srv/work"}"#, false), + ("{}", false), + ] { + let (socket_path, _, handle) = spawn_podman_stub( + "owned-volume", + vec![ + StubResponse::new(StatusCode::NOT_FOUND, ""), + StubResponse::new(StatusCode::CREATED, "{}"), + StubResponse::new( + StatusCode::OK, + format!( + r#"{{"Name":"work","Driver":"local","Options":{options},"Labels":{labels}}}"# + ), + ), + ], + ); + let result = PodmanClient::new(socket_path.clone()) + .create_owned_volume("work", "sandbox-1", "team-a", Some((1234, 1235))) + .await; + assert_eq!(result.is_ok(), accepted, "options {options}: {result:?}"); + handle.await.expect("stub task should finish"); + let _ = std::fs::remove_file(socket_path); + } + } + #[tokio::test] async fn inspect_image_reads_immutable_id_and_oci_config() { let (socket_path, request_log, handle) = spawn_podman_stub( diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 75cccda548..1748a1d96f 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1532,8 +1532,6 @@ pub struct IsolationSpecInput<'a> { pub supervisor_bin: Option<&'a Path>, pub tls_secrets: Option<&'a [String; 3]>, pub identity: &'a openshell_isolation_interface::contract::ResolvedWorkloadIdentity, - /// Whether this workload is created by a rootless Podman service. - pub rootless: bool, } pub struct IsolationSpecs { @@ -1585,46 +1583,19 @@ pub fn build_isolation_specs( .iter() .filter_map(|entry| entry.split_once('=').map(|(key, _)| key.to_string())) .collect(); - if input.image.uses_managed_workspace() - && (input.rootless || input.identity.source == "default") - { - // Podman's archive endpoint leaves named-volume contents owned by - // container root for rootless services and for a rootful USER-less - // image's newly-created workspace. Start the trusted runtime as root - // only long enough to chown the workspace, then irreversibly drop to - // the resolved workload identity before reading bootstrap material or - // accepting a control connection. - workload.command = vec![ - "launch-capability-free".into(), - input.identity.uid.to_string(), - input.identity.gid.to_string(), - crate::isolation::BOOTSTRAP_PATH.into(), - input.image.workspace_root.clone(), - ]; - workload.user = "0:0".into(); - workload.groups.clear(); - workload.cap_drop = vec!["ALL".into()]; - workload.cap_add = vec![ - "CHOWN".into(), - "SETGID".into(), - "SETUID".into(), - "SETPCAP".into(), - ]; - } else { - workload.command = vec![ - "--bootstrap".into(), - crate::isolation::BOOTSTRAP_PATH.into(), - ]; - workload.user.clone_from(&user); - workload.groups = input - .identity - .supplementary_gids - .iter() - .map(ToString::to_string) - .collect(); - workload.cap_drop = vec!["ALL".into()]; - workload.cap_add.clear(); - } + workload.command = vec![ + "--bootstrap".into(), + crate::isolation::BOOTSTRAP_PATH.into(), + ]; + workload.user.clone_from(&user); + workload.groups = input + .identity + .supplementary_gids + .iter() + .map(ToString::to_string) + .collect(); + workload.cap_drop = vec!["ALL".into()]; + workload.cap_add.clear(); workload.apparmor_profile = input .config .app_armor_profile @@ -1934,7 +1905,6 @@ mod tests { supervisor_bin: None, tls_secrets: None, identity: &identity, - rootless: true, }) .unwrap(); for spec in [&specs.workload, &specs.supervisor] { @@ -1942,21 +1912,14 @@ mod tests { assert!(spec.seccomp_profile_path.is_empty()); assert!(spec.no_new_privileges); } - assert_eq!(specs.workload.user, "0:0"); - assert!(specs.workload.groups.is_empty()); - assert_eq!( - specs.workload.cap_add, - vec!["CHOWN", "SETGID", "SETUID", "SETPCAP"] - ); + // The driver creates the managed workspace volume owned by the + // workload identity, so the workload never starts as root. + assert_eq!(specs.workload.user, "1000:1001"); + assert_eq!(specs.workload.groups, vec!["2000"]); + assert!(specs.workload.cap_add.is_empty()); assert_eq!( specs.workload.command, - vec![ - "launch-capability-free", - "1000", - "1001", - crate::isolation::BOOTSTRAP_PATH, - driver_mounts::DEFAULT_WORKSPACE_ROOT, - ] + vec!["--bootstrap", crate::isolation::BOOTSTRAP_PATH] ); assert_eq!(specs.supervisor.user, "1000:1001"); assert_eq!(specs.supervisor.groups, vec!["2000"]); @@ -1985,7 +1948,7 @@ mod tests { "sha256:image".into(), ) .unwrap(); - let rootful_specs = build_isolation_specs(IsolationSpecInput { + let default_specs = build_isolation_specs(IsolationSpecInput { sandbox: &sandbox, config: &config, token_secret: Some("jwt"), @@ -1996,39 +1959,12 @@ mod tests { supervisor_bin: None, tls_secrets: None, identity: &default_identity, - rootless: false, - }) - .unwrap(); - assert_eq!(rootful_specs.workload.user, "0:0"); - assert_eq!( - rootful_specs.workload.command, - vec![ - "launch-capability-free", - "1000", - "1000", - crate::isolation::BOOTSTRAP_PATH, - driver_mounts::DEFAULT_WORKSPACE_ROOT, - ] - ); - let custom_image = resolved_image("sha256:image", "1000:1001", "/workspace/project"); - let custom_specs = build_isolation_specs(IsolationSpecInput { - sandbox: &sandbox, - config: &config, - token_secret: Some("jwt"), - resolver_secret: "resolver", - gpu_devices: None, - requested_image: "image:latest", - image: &custom_image, - supervisor_bin: None, - tls_secrets: None, - identity: &identity, - rootless: true, }) .unwrap(); - assert_eq!(custom_specs.workload.user, "1000:1001"); - assert!(custom_specs.workload.cap_add.is_empty()); + assert_eq!(default_specs.workload.user, "1000:1000"); + assert!(default_specs.workload.cap_add.is_empty()); assert_eq!( - custom_specs.workload.command, + default_specs.workload.command, vec!["--bootstrap", crate::isolation::BOOTSTRAP_PATH] ); let workload_json = serde_json::to_string(&specs.workload).unwrap(); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 7dfd0c41ac..9522c404ba 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1035,7 +1035,12 @@ impl PodmanComputeDriver { let result = async { if managed_workspace { self.client - .create_owned_volume(&vol_name, &sandbox.id, &sandbox.workspace) + .create_owned_volume( + &vol_name, + &sandbox.id, + &sandbox.workspace, + Some((identity.uid, identity.gid)), + ) .await .map_err(ComputeDriverError::from)?; } @@ -1162,7 +1167,6 @@ impl PodmanComputeDriver { supervisor_bin: supervisor_bin_path.as_deref(), tls_secrets: tls_secret_names.as_ref(), identity: &identity, - rootless: self.rootless, }); let mut specs = match specs { Ok(spec) => spec, @@ -1177,7 +1181,7 @@ impl PodmanComputeDriver { let identities = self.validate_user_volume_mounts_available(sandbox).await?; specs.record_resource_identities(&identities)?; self.client - .create_owned_volume(&channel_volume, &sandbox.id, &sandbox.workspace) + .create_owned_volume(&channel_volume, &sandbox.id, &sandbox.workspace, None) .await?; channel_owned.store(true, std::sync::atomic::Ordering::Relaxed); let workload_id = self.client.create_typed_container(&specs.workload).await?; @@ -3040,7 +3044,7 @@ mod tests { assert!( driver .client - .create_owned_volume("private-collision", "sandbox-123", "team-a") + .create_owned_volume("private-collision", "sandbox-123", "team-a", None) .await .is_err() ); @@ -3556,7 +3560,11 @@ mod tests { image_response("sha256:supervisor"), StubResponse::new(StatusCode::NOT_FOUND, ""), // no existing private workspace StubResponse::new(StatusCode::CREATED, "{}"), // workspace volume - owned_volume_response(&container::volume_name(sandbox_id), sandbox_id), + owned_volume_response( + &container::volume_name(sandbox_id), + sandbox_id, + Some((1234, 1235)), // the stub image's OCI user + ), StubResponse::new(StatusCode::CREATED, "{}"), // resolver secret ]; if proxy_secret { @@ -3573,15 +3581,30 @@ mod tests { responses.push(owned_volume_response( &crate::isolation::channel_volume_name(sandbox_id), sandbox_id, + None, )); responses } - fn owned_volume_response(name: &str, sandbox_id: &str) -> StubResponse { + fn owned_volume_response( + name: &str, + sandbox_id: &str, + owner: Option<(u32, u32)>, + ) -> StubResponse { + let options = owner.map_or_else( + || serde_json::json!({}), + |(uid, gid)| { + serde_json::json!({ + "o": format!("uid={uid},gid={gid}"), + "UID": uid.to_string(), + "GID": gid.to_string(), + }) + }, + ); StubResponse::new( StatusCode::OK, serde_json::json!({ - "Name": name, "Driver": "local", "Options": {}, + "Name": name, "Driver": "local", "Options": options, "Labels": {LABEL_SANDBOX_ID: sandbox_id, container::LABEL_SANDBOX_WORKSPACE: ""} }) .to_string(), diff --git a/tests/suites/features/oci-image/tests/oci_image.rs b/tests/suites/features/oci-image/tests/oci_image.rs index 3bd6423252..e5949c3bf8 100644 --- a/tests/suites/features/oci-image/tests/oci_image.rs +++ b/tests/suites/features/oci-image/tests/oci_image.rs @@ -74,19 +74,14 @@ USER 2345:2346 .await; } -/// An image without a `WORKDIR` uses the managed `/sandbox` workspace. An -/// image that declares `USER` provides a `/sandbox` owned by that user. +/// An image without a `WORKDIR` uses the managed `/sandbox` workspace, owned +/// by the image user, even when the image does not contain `/sandbox`. #[tokio::test] async fn default_workdir_uses_managed_workspace() { run("oci-image/default-workdir", async |runner| { let image = TestImage::build( "default-workdir", - &format!( - "FROM {BASE_IMAGE} -RUN mkdir /sandbox && chown 2345:2346 /sandbox && chmod 0700 /sandbox -USER 2345:2346 -" - ), + &format!("FROM {BASE_IMAGE}\nUSER 2345:2346\n"), )?; let checks = workspace_checks("2345:2346", "/sandbox", false); create_sandbox(runner, "default", "dw", &image, &checks)