Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion crates/openshell-core/src/driver_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,22 @@ pub const CONDITION_WORKSPACE_VALIDATION_FAILED: &str = "WorkspaceValidationFail

/// Supervisor exit status reserved for OCI workspace validation failures.
///
/// Local container drivers translate this status into
/// Local container drivers translate a supervisor exit with this status into
/// [`CONDITION_WORKSPACE_VALIDATION_FAILED`] so users receive the specific
/// provisioning failure rather than a generic container exit.
pub const SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED: i32 = 78;

/// Error context for a rejected image-provided OCI working directory.
///
/// The supervisor recognizes it in a failed agent start and exits with
/// [`SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED`].
pub const WORKSPACE_VALIDATION_ERROR_CONTEXT: &str = "image workspace validation failed";

/// Driver condition message for [`CONDITION_WORKSPACE_VALIDATION_FAILED`].
/// Fixed text, because supervisor output may contain secrets.
pub const WORKSPACE_VALIDATION_FAILED_MESSAGE: &str =
"OCI WorkingDir is not usable by the sandbox identity";

/// Ready-condition reason when a container was terminated by an external signal.
///
/// SIGKILL/SIGTERM (exit 137/143) is what a Podman/Docker machine or daemon
Expand Down
62 changes: 50 additions & 12 deletions crates/openshell-driver-docker/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,12 @@ use futures::{Stream, StreamExt};
use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS;
use openshell_core::driver_mounts;
use openshell_core::driver_utils::{
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE,
LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE,
SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh,
temp_extract_container_name, validate_linux_elf_binary,
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_WORKSPACE_VALIDATION_FAILED,
LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME,
LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SANDBOX_RUNTIME_IMAGE_BINARY_PATH,
SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_FAILED_MESSAGE,
extract_first_tar_entry, supervisor_image_should_refresh, temp_extract_container_name,
validate_linux_elf_binary,
};
use openshell_core::gpu::{
CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements,
Expand Down Expand Up @@ -1854,7 +1856,7 @@ impl DockerComputeDriver {
.await;
cleanup_docker_boundary_state(sandbox, &self.config);
return Err(DockerProvisioningFailure::new(
"ControlSupervisorStartFailed",
supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"),
status.message(),
));
}
Expand Down Expand Up @@ -2146,7 +2148,7 @@ impl DockerComputeDriver {
Err(status) => {
handle_docker_runtime_failure(
failure_context,
"ControlSupervisorExited",
supervisor_start_failure_reason(&status, "ControlSupervisorExited"),
format!(
"failed to start Docker control supervisor: {}",
status.message()
Expand Down Expand Up @@ -5286,9 +5288,11 @@ async fn spawn_docker_control_process(
).await;
return;
}
let mut reason = "ControlSupervisorExited";
let mut message = match result {
Some(Ok(status)) => {
warn!(%sandbox_id, status = status.status_code, "Docker supervisor container exited unexpectedly");
reason = supervisor_exit_reason(status.status_code);
format!("Docker supervisor container exited with status {}", status.status_code)
}
Some(Err(error)) => {
Expand Down Expand Up @@ -5319,12 +5323,7 @@ async fn spawn_docker_control_process(
if monitored_shutdown.load(Ordering::Acquire) {
return;
}
handle_docker_runtime_failure(
failure_context,
"ControlSupervisorExited",
message,
)
.await;
handle_docker_runtime_failure(failure_context, reason, message).await;
},
}
});
Expand Down Expand Up @@ -5373,6 +5372,13 @@ async fn wait_for_docker_supervisor_ready(
let state = inspected.state.unwrap_or_default();
match state.health.and_then(|health| health.status) {
Some(HealthStatusEnum::HEALTHY) => return Ok(()),
_ if state.running == Some(false)
&& state.exit_code
== Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)) =>
{
let log_tail = docker_container_log_tail(docker, supervisor_id).await;
return Err(workspace_validation_status(&log_tail));
}
_ if state.running == Some(false) => {
let log_tail = docker_container_log_tail(docker, supervisor_id).await;
let sandbox_log_tail = docker_container_log_tail(docker, sandbox_id).await;
Expand All @@ -5387,6 +5393,38 @@ async fn wait_for_docker_supervisor_ready(
}
}

/// Startup failure for a supervisor that exited because the sandbox rejected
/// the image working directory. The fixed message prefix identifies it for
/// [`supervisor_start_failure_reason`].
fn workspace_validation_status(log_tail: &str) -> Status {
Status::failed_precondition(format!(
"{WORKSPACE_VALIDATION_FAILED_MESSAGE}{}",
format_log_tail(log_tail)
))
}

/// Condition reason for a supervisor that failed before becoming ready.
fn supervisor_start_failure_reason(status: &Status, default: &'static str) -> &'static str {
if status.code() == tonic::Code::FailedPrecondition
&& status
.message()
.starts_with(WORKSPACE_VALIDATION_FAILED_MESSAGE)
{
CONDITION_WORKSPACE_VALIDATION_FAILED
} else {
default
}
}

/// Condition reason for a supervisor that exited with `status_code`.
fn supervisor_exit_reason(status_code: i64) -> &'static str {
if status_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) {
CONDITION_WORKSPACE_VALIDATION_FAILED
} else {
"ControlSupervisorExited"
}
}

fn format_log_tail(log_tail: &str) -> String {
format_named_log_tail("log tail", log_tail)
}
Expand Down
22 changes: 22 additions & 0 deletions crates/openshell-driver-docker/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3595,6 +3595,28 @@ fn docker_oom_kill_stays_terminal_despite_137() {
assert_eq!(ready_reason(&sandbox), CONDITION_EXITED);
}

#[test]
fn supervisor_workspace_validation_exit_is_reported_explicitly() {
let status = workspace_validation_status("image workspace validation failed: denied");
assert!(status.message().contains("log tail: image workspace"));
assert_eq!(
supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"),
CONDITION_WORKSPACE_VALIDATION_FAILED
);
assert_eq!(
supervisor_start_failure_reason(
&Status::failed_precondition("provider SPIFFE socket has no parent directory"),
"ControlSupervisorStartFailed"
),
"ControlSupervisorStartFailed"
);
assert_eq!(
supervisor_exit_reason(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)),
CONDITION_WORKSPACE_VALIDATION_FAILED
);
assert_eq!(supervisor_exit_reason(1), "ControlSupervisorExited");
}

#[test]
fn concurrent_container_removal_is_idempotent() {
let removing = BollardError::DockerResponseServerError {
Expand Down
17 changes: 16 additions & 1 deletion crates/openshell-driver-podman/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ only the channel bootstrap into the existing channel volume, preserving the
workspace. The workload starts before the supervisor so its user namespace exists
when the supervisor joins it; a stopped supervisor resolves that namespace again
on its next start. The driver creates the managed workspace volume owned by
the workload's final UID and GID, so the workload never starts as root.
the workload's final UID and GID, so the workload never starts as root. Custom
image workspaces have no workspace volume or upload.

The runtime must pass the sandbox's unprivileged enforcement probe, including
nested seccomp notification and Landlock. Unsupported runtime defaults fail
Expand Down Expand Up @@ -91,6 +92,20 @@ binary extraction path. `supervisor_image` supplies the dynamically linked
glibc `/openshell-supervisor` binary outside the workload. Image and request
environment belong to agent children, never the supervisor process.

## OCI working directory

OpenShell reads `WORKDIR` from the workload image. If it is unset, `/`, or
`/sandbox`, OpenShell uses its managed `/sandbox` workspace volume. A custom
path must be absolute and normalized, and cannot overlap `/proc`, `/sys`,
`/dev`, OpenShell-reserved paths, or the workload's private control and CA
mounts. Image volumes and driver mounts cannot cover it; mounts nested below it
remain valid.

A custom path stays in the image's container filesystem with its ownership and
permissions. The workload starts as the final non-root user, which must be able
to reach and write the directory. Agent commands use the path as their working
directory.

## Lifecycle and readiness

Create builds both stopped containers and stages the private archives before
Expand Down
26 changes: 24 additions & 2 deletions crates/openshell-driver-podman/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,10 @@ pub struct ContainerState {
/// container-log marker. It is never deserialized from Podman.
#[serde(skip)]
pub startup_diagnostic: Option<String>,
/// Exit status of the sandbox's supervisor companion when it stopped
/// before the workload. It is never deserialized from Podman.
#[serde(skip)]
pub supervisor_exit_code: Option<i64>,
}

#[derive(Debug, Clone, serde::Deserialize)]
Expand Down Expand Up @@ -187,6 +191,10 @@ pub struct ImageConfig {
pub user: String,
#[serde(default)]
pub env: Vec<String>,
#[serde(default)]
pub working_dir: String,
#[serde(default)]
pub volumes: Option<HashMap<String, Value>>,
}

/// Whether a driver-owned volume has exactly the options `OpenShell` creates it
Expand Down Expand Up @@ -1222,12 +1230,12 @@ mod tests {
}

#[tokio::test]
async fn inspect_image_reads_immutable_id_and_oci_user() {
async fn inspect_image_reads_immutable_id_and_oci_config() {
let (socket_path, request_log, handle) = spawn_podman_stub(
"inspect-image",
vec![StubResponse::new(
StatusCode::OK,
r#"{"Id":"sha256:immutable","Config":{"User":"app:staff"}}"#,
r#"{"Id":"sha256:immutable","Config":{"User":"app:staff","Env":["A=one"],"WorkingDir":"/workspace/project","Volumes":{"/workspace/project/cache":{}}}}"#,
)],
);
let client = PodmanClient::new(socket_path.clone());
Expand All @@ -1242,6 +1250,20 @@ mod tests {
image.config.as_ref().map(|config| config.user.as_str()),
Some("app:staff")
);
assert_eq!(
image
.config
.as_ref()
.map(|config| config.working_dir.as_str()),
Some("/workspace/project")
);
assert!(
image
.config
.as_ref()
.and_then(|config| config.volumes.as_ref())
.is_some_and(|volumes| volumes.contains_key("/workspace/project/cache"))
);
handle.await.expect("stub task should finish");
assert_eq!(
request_log
Expand Down
Loading
Loading