From 8ece94e4609494140b77b9e06febd88f7f0cb9ac Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:12:22 -0600 Subject: [PATCH 01/14] test: prepare published condition history acceptance packet --- .../acceptance/condition-history/README.md | 109 +++++ .../condition-history/check_history.py | 49 +++ .../condition-history/fixture/Cargo.toml | 20 + .../fixture/tests/conditions.rs | 395 ++++++++++++++++++ scripts/acceptance/condition-history/run.py | 367 ++++++++++++++++ .../condition-history/test_packet.py | 158 +++++++ 6 files changed, 1098 insertions(+) create mode 100644 scripts/acceptance/condition-history/README.md create mode 100644 scripts/acceptance/condition-history/check_history.py create mode 100644 scripts/acceptance/condition-history/fixture/Cargo.toml create mode 100644 scripts/acceptance/condition-history/fixture/tests/conditions.rs create mode 100644 scripts/acceptance/condition-history/run.py create mode 100644 scripts/acceptance/condition-history/test_packet.py diff --git a/scripts/acceptance/condition-history/README.md b/scripts/acceptance/condition-history/README.md new file mode 100644 index 0000000..b882e4a --- /dev/null +++ b/scripts/acceptance/condition-history/README.md @@ -0,0 +1,109 @@ +# Published condition-history acceptance + +This packet prepares the selected acceptance for [#191](https://github.com/durable-workflow/sdk-rust/issues/191) +and [#192](https://github.com/durable-workflow/sdk-rust/issues/192). +It is source under review. It has not built its consumer or run a container. +Server history review and Security executable review precede a separate General +admission of the exact runner commit, command, fresh capacity and absolute deadline. + +The package is crates.io `durable-workflow =4.12.0`, archive SHA-256 +`620a8287af73c363dac61f0a91f20971a39f3b8cdc9513fbea85cdbd95a74ba4`, +source `4b2440a5618ccc1c98960ea2e1e62127042c3590` (H). Its published tag, +archive and approval records are inputs and remain unchanged. + +`run.py` exports H's official replay consumer and four existing history fixtures +byte for byte, with fixed SHA-256 checks. `fixture/tests/conditions.rs` includes +that consumer and adds the public Worker port of H's four 300/180-second +orphan/tail variants and genuine-timeout refusal. Each case runs with fresh +workers. The typed replay error is observed in the handler before returning the +same error to the Worker; the ordinary published task-failure report is also +retained. The consumer then runs one new live two-member false-wake/reopen case +against unchanged PHP Server 2.11.8, requiring typed `both` completion. + +The original raw proxy artifact for #191 is unavailable. These existing fixtures +are retained evidence, without a claim that they reconstruct that missing full +capture. The live case produces a new capture. #192's original duplicate producer +was the PHP Laravel bridge on native Server 3.0.5. This packet proves installed +Rust replay and does not claim Rust duplicate emission or Server admission behavior. + +All executable image references select immutable `linux/amd64` manifests: + +| Role | Image digest | +| --- | --- | +| Rust 1.86.0 build/consumer | `rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081` | +| PHP Server 2.11.8 | `ghcr.io/durable-workflow/server@sha256:980fe0a052089ced34ec3fcd2be2ea4e77436861d8fb537fb6654cb652a53e07` | +| MySQL | `mysql@sha256:80f4933e3835f9dc4d35a28ec500d7986cb4414e6c6821c5461239cb7beb8995` | +| Redis | `redis@sha256:ca0acbb137c1dc3339c8b147a58fd6f42775d4599327b50e7b116c23de501af2` | + +The Server index is `sha256:3848d5d5cd1b23bd2881e3fdecea64cc44b72e6fb812e577c2db6d0872d065dd`, +source `fa3fc2f2f6b8935ea4a6eea47f6a9221e2adafff`. The MySQL and Redis manifests +were resolved from the existing installed-native pins using digest-verified public +registry index reads. Their indices remain +`sha256:6ea90827b1100f8f2ae306a539f86d2c264a26ed435a2a9f75551dd5c3aeb242` +and `sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499`. + +The build uses two jobs and 3 GiB, with a fresh Cargo home, lock, downloaded crate +and test binary. The public sparse index, lock, archive and VCS commit must agree. +Only dependency resolution/fetch has network access. Compilation is offline. +There is no SDK source replacement or reuse of a prior acceptance binary/cache. + +Runtime starts after all build containers are absent. Loopback replay runs alone. +The PHP case has at most five roles: MySQL 1 CPU/1.25 GiB, Redis 0.25 CPU/128 MiB, +HTTP 0.75 CPU/768 MiB, queue 0.5 CPU/512 MiB, and a combined Worker/driver +0.5 CPU/768 MiB. Bootstrap is removed before HTTP and queue startup. Total runtime +caps are 3 CPU/3.375 GiB. There is no repair or proxy role. Each role retains the +accepted installed-native supervisor, private stdin-to-tmpfs credential delivery, +dual ownership labels, evidence fence and cleanup. Database/cache data is tmpfs; +image volume declarations must be fully covered before any runtime starts. +Docker uses an empty registry configuration and the existing local Unix daemon. +No host credential, Docker socket or published port is mounted into a container. + +One 30-minute absolute owner window includes setup, fetch, compilation, execution +and cleanup. Active work ends 60 seconds before the final ceiling. Each role in +the one Cell shares a maximum 600-second deadline, followed by TERM/KILL. +A detached owner recovery process is armed before pulls or build. It removes only +the exact owner/run labels and purges the interrupted Cell tree, since credential +values intentionally never enter `run.json`. It is canceled only after direct +zero-survivor readbacks. A failed attempt is retained as failed; no edit, retry, +fallback or extension is part of this command. + +After the exact packet is reviewed and admitted, substitute the reviewed runner +commit R, a fresh absolute start S, an unused private work directory W and a +16-hex owner token T in this one command: + +```sh +python3 scripts/acceptance/condition-history/run.py run \ + --package-commit 4b2440a5618ccc1c98960ea2e1e62127042c3590 \ + --runner-commit R --work-dir W --run-token T \ + --started-unix S --active-unix ACTIVE --cleanup-unix CLEANUP +``` + +`ACTIVE` must equal `S + 1740`; `CLEANUP` must equal `S + 1800`. The runner refuses +wrong H, a dirty/wrong R, an existing work directory or stale/malformed deadlines +before harness command construction or launch. Host/capacity facts and actual +private paths belong in task evidence, not public GitHub prose. + +If the client dies, the same reviewed source supports immediate standalone cleanup: + +```sh +python3 scripts/acceptance/condition-history/run.py recover --work-dir W --run-token T +``` + +The summary retains source/input/lock/package/binary identities, actual toolchain, +role ownership/resource readbacks, typed replay outputs, fresh raw live history and +an offline lineage comparator. The comparator has adversarial lineage/order/timer +controls. Its fetch transport is the public SDK raw control-plane method; it is +not an independent second HTTP client. After execution, a separate read-only owner +audit must confirm all matching containers, networks, volumes and processes absent. +Publish a sanitized immutable result before acceptance or closure. + +Inert source checks, which do not invoke Docker or Cargo: + +```sh +python3 -m unittest discover -s scripts/acceptance/condition-history -p test_packet.py +rustfmt --edition 2021 --check scripts/acceptance/condition-history/fixture/tests/conditions.rs +``` + +These check source guards, fixed fixture export and the history comparator. They +do not prove that the new Rust consumer compiles or that either live acceptance +criterion passes. Those remain subject to the reviewed bounded execution. diff --git a/scripts/acceptance/condition-history/check_history.py b/scripts/acceptance/condition-history/check_history.py new file mode 100644 index 0000000..c8c0436 --- /dev/null +++ b/scripts/acceptance/condition-history/check_history.py @@ -0,0 +1,49 @@ +"""Offline comparator for the newly captured two-member PHP history.""" + + +def require(condition, label): + if not condition: + raise ValueError(label) + + +def check(report): + require(report.get('outcome') == 'pass' and report.get('result') == 'both', 'typed-result') + snapshots = [report.get(key) for key in ('initial', 'reopened', 'history')] + require(all(isinstance(events, list) for events in snapshots), 'history-arrays') + initial, reopened, final = snapshots + require(reopened[:len(initial)] == initial and final[:len(reopened)] == reopened, 'append-only-history') + opens = [e for e in final if e.get('event_type') == 'ConditionWaitOpened'] + require(len(opens) >= 4, 'four-opens') + first = [e['payload'] for e in opens[:4]] + require(all(type(p.get(field)) is int for p in first + for field in ('sequence', 'parallel_group_index', 'parallel_group_base_sequence', 'parallel_group_size')), + 'typed-group-numbers') + require([p.get('sequence') for p in first] == [1, 2, 3, 4], 'open-sequences') + require([p.get('condition_key') for p in first] == ['votes', 'approvals', 'votes', 'approvals'], 'member-keys') + require([p.get('parallel_group_index') for p in first] == [0, 1, 0, 1], 'member-indices') + require(all(p.get('parallel_group_id') == 'parallel-calls:1:2' + and p.get('parallel_group_kind') == 'condition' + and p.get('parallel_group_base_sequence') == 1 + and p.get('parallel_group_size') == 2 for p in first), 'group-identity') + for index in (0, 1): + original, successor = first[index], first[index + 2] + for field in ('condition_wait_occurrence_id', 'condition_definition_fingerprint'): + require(isinstance(original.get(field), str) and original[field] == successor.get(field), field) + require(original.get('timeout_seconds') == 300 + and isinstance(successor.get('timeout_seconds'), int) + and not isinstance(successor['timeout_seconds'], bool) + and 0 <= successor['timeout_seconds'] <= 300, 'original-timeout-bound') + require(first[0]['condition_wait_occurrence_id'] != first[1]['condition_wait_occurrence_id'], 'distinct-members') + types = [e.get('event_type') for e in final] + require(not any(t in ('WorkflowFailed', 'ReplayFailure', 'WorkflowTimedOut') for t in types), 'no-replay-failure') + require(types.count('WorkflowCompleted') == 1, 'one-completion') + signal_rows = [(i, e.get('payload', {}).get('signal_name')) for i, e in enumerate(final) + if e.get('event_type') == 'SignalReceived'] + require([name for _, name in signal_rows] == ['vote', 'vote', 'approve'], 'signal-order') + open_rows = [i for i, e in enumerate(final) if e.get('event_type') == 'ConditionWaitOpened'] + require(open_rows[1] < signal_rows[0][0] < open_rows[2] < open_rows[3] < signal_rows[1][0], 'false-wake-row-order') + early_satisfied = [e.get('payload', {}).get('sequence') for e in reopened + if e.get('event_type') == 'ConditionWaitSatisfied'] + require(2 in early_satisfied and 1 not in early_satisfied, 'unresolved-first-member') + return {'outcome': 'pass', 'first_open_sequences': [1, 2], 'reopened_sequences': [3, 4], + 'captured_events': len(final), 'result': report['result']} diff --git a/scripts/acceptance/condition-history/fixture/Cargo.toml b/scripts/acceptance/condition-history/fixture/Cargo.toml new file mode 100644 index 0000000..ae960e9 --- /dev/null +++ b/scripts/acceptance/condition-history/fixture/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "installed-condition-history" +version = "0.0.0" +edition = "2021" +publish = false +rust-version = "1.86" + +[workspace] + +[dependencies] +durable-workflow = "=4.12.0" +chrono = { version = "0.4", default-features = false, features = ["alloc"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time"] } + +[profile.dev] +debug = 0 +incremental = false diff --git a/scripts/acceptance/condition-history/fixture/tests/conditions.rs b/scripts/acceptance/condition-history/fixture/tests/conditions.rs new file mode 100644 index 0000000..d52be5c --- /dev/null +++ b/scripts/acceptance/condition-history/fixture/tests/conditions.rs @@ -0,0 +1,395 @@ +// run.py exports this file byte-for-byte from the immutable package source H. +// It supplies the official Worker/loopback transport, not SDK implementation. +include!("official.rs"); + +mod installed_conditions { + use super::*; + use durable_workflow::{RawRequest, WorkflowResultOptions}; + + type Outcome = Result>; + + fn check(ok: bool, message: &str) -> Outcome<()> { + if ok { + Ok(()) + } else { + Err(message.into()) + } + } + + fn event(kind: &str, at: &str, payload: Value) -> Value { + json!({"event_type": kind, "timestamp": at, "payload": payload}) + } + + // Faithful public-Worker port of H's decision_history: sequences, row order, + // occurrence identity, timestamps and timeout values are kept unchanged. + fn decision_history(reopen: u64, genuine_timeout: bool, tail_fire: bool) -> Vec { + let wait = |seq: u64, at: &str, timeout: u64| { + event( + "ConditionWaitOpened", + at, + json!({ + "sequence": seq, "condition_wait_id": format!("condition:{seq}"), + "condition_wait_occurrence_id": "rust:condition-wait:0", + "condition_key": "decision-1", "condition_definition_fingerprint": "sha256:decision-v1", + "timeout_seconds": timeout + }), + ) + }; + let timer = |kind: &str, seq: u64, at: &str, delay: u64| { + event( + kind, + at, + json!({ + "sequence": seq, "timer_id": format!("condition-timer:{seq}"), + "timer_kind": "condition_timeout", "condition_wait_id": format!("condition:{seq}"), + "delay_seconds": delay + }), + ) + }; + let orphan = timer("TimerFired", 1, "2026-11-04T04:12:36.956000+00:00", 300); + let mut history = vec![ + wait(1, "2026-11-04T04:07:36.890000+00:00", 300), + timer("TimerScheduled", 1, "2026-11-04T04:07:36.891000+00:00", 300), + ]; + if genuine_timeout { + history.push(orphan.clone()); + } + history.extend([ + wait(2, "2026-11-04T04:07:36.908000+00:00", 300), + timer("TimerScheduled", 2, "2026-11-04T04:07:36.909000+00:00", 300), + event("SignalReceived", "2026-11-04T04:09:36.979000+00:00", json!({ + "signal_name": "decision", "signal_id": "s1", "workflow_command_id": "command-s1", + "arguments": encode_payload(&json!(["proposal-1"]), DEFAULT_CODEC).unwrap() + })), + event("ConditionWaitSatisfied", "2026-11-04T04:09:37.008000+00:00", json!({ + "sequence": 2, "condition_wait_id": "condition:2", + "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "decision-1", + "condition_definition_fingerprint": "sha256:decision-v1" + })), + timer("TimerCancelled", 2, "2026-11-04T04:09:37.009000+00:00", 300), + wait(3, "2026-11-04T04:09:37.010000+00:00", reopen), + timer("TimerScheduled", 3, "2026-11-04T04:09:37.011000+00:00", reopen), + ]); + if !genuine_timeout { + history.push(orphan); + } + if tail_fire { + history.push(timer( + "TimerFired", + 3, + "2026-11-04T04:12:37.216000+00:00", + reopen, + )); + } + history + } + + async fn decision_replay(history: Vec, delivery: &str) -> Outcome { + let id = format!("installed-orphan-{delivery}"); + let task = json!({"task_id": id, "workflow_id": "installed-orphan", "run_id": "installed-run", + "workflow_type": "installed.decision", "arguments": encode_payload(&json!([]), DEFAULT_CODEC)?, + "payload_codec": DEFAULT_CODEC, "history_events": history, "workflow_task_attempt": 1, + "lease_owner": "regression-corpus-worker"}); + let server = FixtureServer::start(task, 0, None); + let client = Client::builder(server.base_url()) + .timeout(Duration::from_secs(2)) + .build()?; + let typed_error = Arc::new(Mutex::new(None::)); + let captured_error = Arc::clone(&typed_error); + let mut worker = + Worker::new(client, "regression-corpus").worker_id("regression-corpus-worker"); + worker.register_workflow("installed.decision", move |ctx, _| { + let captured_error = Arc::clone(&captured_error); + async move { + let outcome = match ctx + .wait_condition( + ConditionWaitOptions::new("decision-1", "sha256:decision-v1") + .timeout(Duration::from_secs(300)), + || Ok(false), + ) + .await + { + Err(Error::NonDeterministicReplay(failure)) => { + *captured_error.lock().unwrap() = Some(json!({"reason": failure.reason, + "sequence": failure.sequence})); + return Err(Error::NonDeterministicReplay(failure)); + } + result => result?, + }; + Ok(json!(if outcome.is_satisfied() { + "satisfied" + } else { + "timed_out" + })) + } + }); + check( + worker.run_once().await? == 1, + "one history task must settle", + )?; + let completion = server.request_body(&format!("/api/worker/workflow-tasks/{id}/complete")); + let failure = server.request_body(&format!("/api/worker/workflow-tasks/{id}/fail")); + check( + completion.is_some() != failure.is_some(), + "exactly one task report is required", + )?; + let captured = typed_error.lock().unwrap().clone(); + Ok(json!({"completion": completion, "failure": failure, "typed_error": captured})) + } + + #[tokio::test] + async fn installed_history_replay() -> Outcome<()> { + let mut results = serde_json::Map::new(); + for name in [ + "two-member-condition-group-server-reopen.json", + "timer-separated-condition-group-server-reopen.json", + "cooperative-grouped-condition-reopen.json", + "condition-orphaned-duplicate-timeout-keeps-wait.json", + ] { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/replay-regressions") + .join(name); + let fixture: Value = serde_json::from_slice(&fs::read(path)?)?; + let a = execute_fixture_delivery(&fixture, "installed-a").await?; + let b = execute_fixture_delivery(&fixture, "installed-b").await?; + check( + a == b, + "fresh workers must produce identical canonical replay results", + )?; + results.insert(name.to_string(), a); + } + for reopen in [300, 180] { + for tail in [false, true] { + let label = format!("reopen-{reopen}-tail-{tail}"); + let history = decision_history(reopen, false, tail); + let a = decision_replay(history.clone(), "a").await?; + let b = decision_replay(history.clone(), "b").await?; + check(a == b, "fresh workers must preserve orphan/tail results")?; + check( + a["failure"].is_null(), + "valid orphan history must not fail replay", + )?; + let commands = a["completion"]["commands"] + .as_array() + .ok_or("missing command list")?; + check( + commands.len() == 1, + "orphan/tail replay must produce exactly one command", + )?; + if tail { + check( + commands[0]["type"] == "complete_workflow", + "tail timer must complete as timed out", + )?; + let envelope: PayloadEnvelope = + serde_json::from_value(commands[0]["result"].clone())?; + let value: Value = decode_payload(&envelope)?; + check( + value == "timed_out", + "tail result must be the typed timed-out value", + )?; + } else { + check( + commands[0]["type"] == "open_condition_wait" + && commands[0]["timeout_seconds"] == 0, + "late orphan must keep logical wait open with zero remaining time", + )?; + } + results.insert(label, json!({"input_history": history, "observed": a})); + } + } + let history = decision_history(300, true, false); + let negative = decision_replay(history.clone(), "negative-a").await?; + let repeat = decision_replay(history.clone(), "negative-b").await?; + check( + negative == repeat, + "genuine-timeout refusal must survive fresh workers", + )?; + check( + negative["completion"].is_null(), + "genuine timeout must not publish completion", + )?; + check( + negative["typed_error"]["reason"] == "condition_wait_reopened_after_timeout" + && negative["typed_error"]["sequence"] == 2 + && negative["failure"]["failure"]["type"] == "RustWorkflowTaskFailure", + "genuine-timeout reopen must retain its typed refusal reason", + )?; + results.insert( + "genuine-timeout".into(), + json!({"input_history": history, "observed": negative}), + ); + fs::write( + "/state/replay.json", + serde_json::to_vec_pretty(&json!({"outcome": "pass", "cases": results}))?, + )?; + Ok(()) + } + + async fn history(client: &Client, workflow: &str, run: &str) -> Outcome> { + let mut events = Vec::new(); + let mut next = None; + for _ in 0..50 { + let mut request = RawRequest::get(["workflows", workflow, "runs", run, "history"]); + if let Some(token) = next.take() { + request = request.query("next_page_token", token); + } + let response = client.control_plane_request(request).await?; + check(response.status() == 200, "history read must succeed")?; + let page = response.json()?; + events.extend( + page["events"] + .as_array() + .ok_or("history has no event list")? + .iter() + .cloned(), + ); + next = page["next_page_token"] + .as_str() + .filter(|s| !s.is_empty()) + .map(str::to_owned); + if next.is_none() { + return Ok(events); + } + } + Err("history pagination did not end".into()) + } + + async fn wait_history( + client: &Client, + workflow: &str, + run: &str, + opens: usize, + ) -> Outcome> { + for _ in 0..300 { + let events = history(client, workflow, run).await?; + check( + !events.iter().any(|e| { + matches!( + e["event_type"].as_str(), + Some("WorkflowFailed" | "ReplayFailure") + ) + }), + "live workflow must not record a replay failure", + )?; + if events + .iter() + .filter(|e| e["event_type"] == "ConditionWaitOpened") + .count() + >= opens + { + return Ok(events); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err("condition opens did not arrive within the bounded wait".into()) + } + + #[tokio::test] + #[ignore = "requires separately admitted unchanged published PHP 2.11.8 runtime"] + async fn installed_live_group() -> Outcome<()> { + let token = Some(fs::read_to_string("/credentials/token")?); + let operator = Client::builder("http://qualification-origin:8080") + .control_token(token.clone()) + .build()?; + let mut ready = false; + for _ in 0..120 { + if let Ok(response) = operator + .control_plane_request(RawRequest::get(["health"]).timeout(Duration::from_secs(2))) + .await + { + if response.status() == 200 { + ready = true; + break; + } + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + check(ready, "PHP runtime readiness deadline")?; + let cluster = operator + .control_plane_request(RawRequest::get(["cluster", "info"])) + .await?; + check( + cluster.status() == 200, + "cluster identity read must succeed", + )?; + check( + cluster.json()?["version"] == "2.11.8", + "wrong PHP Server version", + )?; + let client = Client::builder("http://qualification-origin:8080") + .worker_token(token) + .build()?; + let mut worker = Worker::new(client, "installed-condition-history") + .worker_id("installed-condition-history") + .max_concurrent_workflow_tasks(1) + .poll_timeout(Duration::from_millis(100)); + worker.register_workflow("installed.two-member", |ctx, _| async move { + let votes = ctx.clone(); + let approvals = ctx.clone(); + ctx.parallel(vec![ + ParallelOperation::condition( + ConditionWaitOptions::new("votes", "sha256:votes-v1") + .timeout(Duration::from_secs(300)), + move || Ok(votes.signals("vote")?.len() >= 2), + ), + ParallelOperation::condition( + ConditionWaitOptions::new("approvals", "sha256:approvals-v1") + .timeout(Duration::from_secs(300)), + move || Ok(!approvals.signals("approve")?.is_empty()), + ), + ]) + .await?; + Ok(json!("both")) + }); + let (stop, stopped) = tokio::sync::oneshot::channel::<()>(); + let execution = tokio::spawn(async move { + worker + .run_until(async { + let _ = stopped.await; + }) + .await + }); + let driver = async { + let handle = operator + .start_workflow( + "installed.two-member", + "installed-condition-history", + "installed-two-member", + json!([]), + ) + .await?; + let run = handle + .run_id + .as_deref() + .ok_or("start omitted selected run")?; + let initial = wait_history(&operator, &handle.workflow_id, run, 2).await?; + handle.signal("vote", json!(["one"])).await?; + let reopened = wait_history(&operator, &handle.workflow_id, run, 4).await?; + handle.signal("vote", json!(["two"])).await?; + handle.signal("approve", json!(["yes"])).await?; + let value = handle + .result(WorkflowResultOptions { + poll_interval: Duration::from_millis(100), + timeout: Duration::from_secs(30), + }) + .await?; + check( + value == "both", + "two-member workflow must return typed both", + )?; + let final_history = history(&operator, &handle.workflow_id, run).await?; + fs::write( + "/state/live.json", + serde_json::to_vec_pretty(&json!({"outcome": "pass", + "workflow_id": handle.workflow_id, "run_id": run, "result": value, + "initial": initial, "reopened": reopened, "history": final_history}))?, + )?; + Ok::<_, Box>(()) + } + .await; + let _ = stop.send(()); + execution.await??; + driver + } +} diff --git a/scripts/acceptance/condition-history/run.py b/scripts/acceptance/condition-history/run.py new file mode 100644 index 0000000..cd7c39a --- /dev/null +++ b/scripts/acceptance/condition-history/run.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""One explicitly admitted public-package condition-history acceptance. + +Source preparation and stubbed tests do not authorize invoking this runner. +See README.md for its fixed tuple, command and remaining review gate. +""" +import argparse +import importlib.util +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +import time +import tomllib +import types +import urllib.request + +HERE = pathlib.Path(__file__).resolve().parent +REPO = HERE.parents[2] +spec = importlib.util.spec_from_file_location('installed_native', HERE.parent / 'installed-native/run.py') +native = importlib.util.module_from_spec(spec) +spec.loader.exec_module(native) +spec = importlib.util.spec_from_file_location('condition_history_check', HERE / 'check_history.py') +comparator = importlib.util.module_from_spec(spec) +spec.loader.exec_module(comparator) + +H = '4b2440a5618ccc1c98960ea2e1e62127042c3590' +CRATE_SHA = '620a8287af73c363dac61f0a91f20971a39f3b8cdc9513fbea85cdbd95a74ba4' +RUST = 'rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081' +SERVER = 'ghcr.io/durable-workflow/server@sha256:980fe0a052089ced34ec3fcd2be2ea4e77436861d8fb537fb6654cb652a53e07' +SERVER_INDEX = 'sha256:3848d5d5cd1b23bd2881e3fdecea64cc44b72e6fb812e577c2db6d0872d065dd' +SERVER_SOURCE = 'fa3fc2f2f6b8935ea4a6eea47f6a9221e2adafff' +MYSQL = 'mysql@sha256:80f4933e3835f9dc4d35a28ec500d7986cb4414e6c6821c5461239cb7beb8995' +REDIS = 'redis@sha256:ca0acbb137c1dc3339c8b147a58fd6f42775d4599327b50e7b116c23de501af2' +INPUTS = { + 'tests/replay_regression_corpus.rs': 'b54be69c5b82ec6ad32bd2871c3451f182185fa01ca3316a1c3aee935c48c9cc', + 'tests/fixtures/replay-regressions/two-member-condition-group-server-reopen.json': + '815526c761ce314960335b0f7cafaa0aa60fa519ddd08252451a7a5ebdca821d', + 'tests/fixtures/replay-regressions/timer-separated-condition-group-server-reopen.json': + 'f86ecb0cc57634a3eb4c87ffbd4fa55555b82c2794a16cd7b4ce9d33e009d0f6', + 'tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json': + '50a630596b5d85a392a33cc595fe58a513f59f6ebdbba928fa22be4ea0fb9075', + 'tests/fixtures/replay-regressions/condition-orphaned-duplicate-timeout-keeps-wait.json': + '012790d1fc23fe11ef9208d3a8243dc282d160903f7df98396c77e1f8fd898f7', +} +CELL = 'condition-history' +PREFIX = 'sdk-rust-condition' +LIMITS = {'mysql': ('1', '1280m'), 'redis': ('0.25', '128m'), + 'bootstrap': ('0.75', '768m'), 'runtime': ('0.75', '768m'), + 'queue': ('0.5', '512m'), 'consumer': ('0.5', '768m')} + + +def require(condition, label): + if not condition: + raise SystemExit(label) + + +def git(*args): + return native.call(['git', '-C', str(REPO), *args]).stdout.strip() + + +def preflight(args): + # These are explicit checks: Python optimization cannot remove admission. + require(args.package_commit == H, 'wrong fixed package source') + require(re.fullmatch('[0-9a-f]{40}', args.runner_commit), 'invalid runner commit') + require(git('rev-parse', 'HEAD') == args.runner_commit, 'wrong runner head') + require(not git('status', '--porcelain', '--untracked-files=all'), 'runner checkout is not clean') + require(git('rev-parse', 'refs/tags/4.12.0^{commit}') == H, 'wrong immutable source tag') + require(not os.path.lexists(args.work_dir), 'work directory already exists') + require(re.fullmatch('[0-9a-f]{16}', args.run_token), 'invalid owner token') + require(args.cleanup_unix == args.started_unix + 1800 + and args.active_unix == args.cleanup_unix - 60, 'wrong absolute deadline envelope') + require(args.started_unix <= time.time() < args.started_unix + 30, 'stale launch timestamp') + # No command construction, pull, output directory or Popen precedes this. + + +def remaining(args): + seconds = int(args.active_unix - time.time()) + require(seconds > 10, 'active deadline exhausted') + return seconds + + +def command(args, argv, **kwargs): + return native.call(argv, timeout=min(kwargs.pop('timeout', 120), remaining(args)), **kwargs) + + +def export_consumer(fixture): + shutil.copytree(HERE / 'fixture', fixture) + for source, expected in INPUTS.items(): + raw = native.call(['git', '-C', str(REPO), 'show', H + ':' + source], text=False).stdout + require(native.hashlib.sha256(raw).hexdigest() == expected, 'fixed history input differs') + target = fixture / ('tests/official.rs' if source == 'tests/replay_regression_corpus.rs' else source) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(raw) + + +def build(args, fixture, owner, script, phase, network=None): + name = owner.base + '-' + phase + limit = remaining(args) - 10 + line = ['docker', 'run', '--rm', '--init', '--name', name, *owner.labels(name), + '--platform', 'linux/amd64', '--cpus', '2', '--memory', '3g', '--pids-limit', '1024', + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', + '--user', '%d:%d' % (os.getuid(), os.getgid()), + '--mount', 'type=bind,src=%s,dst=/work' % fixture, '-w', '/work', + '-e', 'CARGO_HOME=/work/.cargo', '-e', 'CARGO_BUILD_JOBS=2', + '-e', 'CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback'] + if network: + line += ['--network', network] + try: + return command(args, line + [RUST, 'timeout', '--kill-after=10s', str(limit) + 's', 'sh', '-ec', script], + timeout=limit + 10) + finally: + owner.best_effort('build-remove', phase, native.remove_owned_container, name, owner) + + +def verify_crate(fixture): + lock = tomllib.loads((fixture / 'Cargo.lock').read_text()) + packages = [p for p in lock['package'] if p['name'] == 'durable-workflow'] + require(len(packages) == 1 and packages[0]['version'] == '4.12.0' + and packages[0]['source'] == native.CRATES_IO_SOURCE + and packages[0]['checksum'] == CRATE_SHA, 'wrong public package lock') + archives = list((fixture / '.cargo/registry/cache').glob('*/durable-workflow-4.12.0.crate')) + require(len(archives) == 1 and native.sha256(archives[0]) == CRATE_SHA, 'wrong downloaded archive') + with native.tarfile.open(archives[0]) as archive: + vcs = json.load(archive.extractfile('durable-workflow-4.12.0/.cargo_vcs_info.json')) + require(vcs['git']['sha1'] == H and not vcs['git'].get('dirty'), 'wrong public package VCS') + require(not (fixture / '.cargo/config.toml').exists(), 'registry replacement is forbidden') + return {'version': '4.12.0', 'source': H, 'archive_sha256': CRATE_SHA, + 'lock_sha256': native.sha256(fixture / 'Cargo.lock')} + + +def registry_identity(): + request = urllib.request.Request('https://index.crates.io/du/ra/durable-workflow', + headers={'User-Agent': 'durable-workflow-condition-history-acceptance'}) + with urllib.request.urlopen(request, timeout=30) as response: + versions = [json.loads(line) for line in response.read(8 * 1024 * 1024).splitlines()] + exact = [version for version in versions if version.get('vers') == '4.12.0'] + require(len(exact) == 1 and exact[0].get('cksum') == CRATE_SHA + and exact[0].get('yanked') is False, 'public registry identity refused') + return {'version': '4.12.0', 'sha256': CRATE_SHA, 'yanked': False} + + +class ConditionCell(native.Cell): + def __init__(self, *args): + super().__init__(*args) + self.receipts = [] + + def launch(self, line, name, image, child, user, credentials): + role = name.removeprefix(self.task + '-') + require(role in LIMITS, 'unknown condition role') + require(len(self.containers) < 5, 'condition role ceiling reached') + cpus, memory = LIMITS[role] + line[line.index('--cpus') + 1] = cpus + line[line.index('--memory') + 1] = memory + line += ['--platform', 'linux/amd64'] + if role in ('bootstrap', 'runtime', 'queue'): + line += ['--mount', 'type=bind,src=%s,dst=/state' % self.state] + name = super().launch(line, name, image, child, user, credentials) + # Targeted readback deliberately excludes environment and host paths. + template = ('{"id":{{json .Id}},"image":{{json .Image}},"memory":{{.HostConfig.Memory}},' + '"nano_cpus":{{.HostConfig.NanoCpus}},"pid":{{.State.Pid}},' + '"run":{{json (index .Config.Labels "' + native.RUN_LABEL + '")}},' + '"owner":{{json (index .Config.Labels "' + native.OWNER_LABEL + '")}},' + '"mounts":"{{range .Mounts}}{{.Type}}:{{.Destination}};{{end}}"}') + receipt = json.loads(native.call(['docker', 'inspect', '--format', template, name]).stdout) + require(receipt['run'] == self.harness_run.token and receipt['owner'] == self.harness_run.prefix, + 'role readback ownership mismatch') + require(receipt['memory'] == int(memory.removesuffix('m')) * 1024 * 1024 + and receipt['nano_cpus'] == int(float(cpus) * 1_000_000_000), 'role resource mismatch') + require('volume:' not in receipt['mounts'], 'anonymous volumes are forbidden') + receipt.update(role=role, deadline_unix=self.deadline, credentials=credentials) + self.receipts.append(receipt) + return name + + def start_php(self, token): + env = {'APP_ENV': 'testing', 'APP_DEBUG': 'false', 'DB_CONNECTION': 'mysql', + 'DB_HOST': 'mysql', 'DB_DATABASE': 'durable_workflow', 'DB_USERNAME': 'workflow', + 'DB_PASSWORD': 'workflow', 'REDIS_HOST': 'redis', 'QUEUE_CONNECTION': 'redis', + 'CACHE_STORE': 'redis', 'DW_AUTH_DRIVER': 'token', 'DW_WORKER_PROTOCOL_VERSION': '1.20'} + mysql = self.plain('mysql', MYSQL, [], alias='mysql', tmpfs='/var/lib/mysql', pids=512, + caps=('CHOWN', 'DAC_OVERRIDE', 'SETGID', 'SETUID'), + env={'MYSQL_DATABASE': 'durable_workflow', 'MYSQL_USER': 'workflow', + 'MYSQL_PASSWORD': 'workflow', 'MYSQL_ROOT_PASSWORD': 'root'}) + self.plain('redis', REDIS, [], alias='redis', tmpfs='/data', caps=('CHOWN', 'SETGID', 'SETUID')) + until = min(time.time() + 120, self.deadline - 15) + while native.call(['docker', 'exec', mysql, 'mysqladmin', 'ping', '-h', 'localhost', '-uworkflow', '-pworkflow'], + check=False).returncode: + require(time.time() < until, 'mysql readiness deadline') + time.sleep(1) + bootstrap = self.plain('bootstrap', SERVER, ['server-bootstrap'], env=env, + caps=('DAC_OVERRIDE',), credentials='server') + require(native.call(['docker', 'wait', bootstrap], timeout=max(1, self.deadline - time.time())).stdout.strip() == '0', + 'PHP bootstrap failed') + native.remove_owned_container(bootstrap, self.harness_run) + self.containers.remove(bootstrap) + self.plain('runtime', SERVER, [], env=env, alias='qualification-origin', credentials='both', + caps=('CHOWN', 'DAC_OVERRIDE', 'KILL', 'SETGID', 'SETUID')) + self.plain('queue', SERVER, ['php', 'artisan', 'queue:work', '--sleep=1', '--tries=3', '--max-time=600'], + env=env, user='1000:1000', credentials='server') + + def execute(self, args): + outcome = {'outcome': 'fail', 'deadline_unix': self.deadline} + try: + self.cell_path('state').mkdir(parents=True) + self.dir.chmod(0o777) + self.state.chmod(0o777) + command(args, ['docker', 'network', 'create', '--internal', *self.harness_run.labels(self.task), self.task]) + # No service roles exist during loopback replay; then exactly four + # service roles plus one combined driver/Worker. No proxy/repair. + replay = self.start('consumer', RUST, ['/fixture/condition-history', '--exact', + 'installed_conditions::installed_history_replay', '--nocapture']) + require(command(args, ['docker', 'wait', replay], timeout=300).stdout.strip() == '0', 'installed replay failed') + native.remove_owned_container(replay, self.harness_run) + self.containers.remove(replay) + self.start_php(self.token) + consumer = self.start('consumer', RUST, ['/fixture/condition-history', '--exact', + 'installed_conditions::installed_live_group', '--ignored', '--nocapture'], credentials='auth') + require(command(args, ['docker', 'wait', consumer], timeout=180).stdout.strip() == '0', 'live group failed') + finally: + stopped = self.remove_docker_objects() + try: + require(stopped, 'writer cleanup uncertain') + self.fence_artifacts() + require(not self.harness_run.cleanup_errors, 'evidence fence failed') + replay = json.loads(self.artifact_bytes(self.state / 'replay.json')) + live = json.loads(self.artifact_bytes(self.state / 'live.json')) + require(replay.get('outcome') == 'pass' and len(replay.get('cases', {})) == 9, 'replay report incomplete') + outcome.update(outcome='pass', replay=replay, live=live, history_check=comparator.check(live), + role_receipts=self.receipts) + except BaseException: + outcome['outcome'] = 'fail' + self.harness_run.best_effort('evidence-purge', CELL, self.purge_evidence) + raise + return outcome + + +def recover(root, token): + record = json.loads(native.confined_child(root, 'run.json').read_text()) + require(record.get('owner') == PREFIX and record.get('token') == token + and record.get('cells') == [CELL], 'wrong recovery ownership') + owner = native.Run(PREFIX, token=token) + result = native.recover_run(root, owner, [CELL]) + require(result['pass'], 'recovery has survivors or uncertainty') + return result + + +def execute(args): + preflight(args) + args.work_dir.mkdir(mode=0o700) + owner = native.Run(PREFIX, token=args.run_token) + record = {'owner': PREFIX, 'token': owner.token, 'cells': [CELL], 'package_commit': H, + 'runner_commit': args.runner_commit, 'started_unix': args.started_unix, + 'active_unix': args.active_unix, 'cleanup_unix': args.cleanup_unix} + native.write_owner_only(args.work_dir / 'run.json', json.dumps(record)) + endpoint = native.call(['docker', 'context', 'inspect', '--format', '{{.Endpoints.docker.Host}}']).stdout.strip() + require(endpoint.startswith('unix://'), 'only the admitted local Docker daemon is supported') + public_config = args.work_dir / 'public-docker' + public_config.mkdir(mode=0o700) + native.write_owner_only(public_config / 'config.json', '{"auths":{}}\n') + # Keep the local daemon endpoint while removing registry logins and helpers. + # The endpoint is private evidence and is never serialized into the summary. + os.environ['DOCKER_HOST'] = endpoint + os.environ['DOCKER_CONFIG'] = str(public_config) + # The detached recovery process survives this client's death. Every Docker + # role also enforces its own absolute TERM/KILL deadline independently. + watchdog = subprocess.Popen([sys.executable, str(HERE / 'run.py'), 'recover', '--work-dir', str(args.work_dir), + '--run-token', owner.token, '--at', str(args.cleanup_unix)], start_new_session=True, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + result = {'outcome': 'fail', 'run': record} + try: + registry = registry_identity() + images = [RUST, SERVER, MYSQL, REDIS] + for image in images: + command(args, ['docker', 'pull', '--platform', 'linux/amd64', image], timeout=600) + identities = {image: native.image_identity(image) for image in images} + require(all(i['os'] == 'linux' and i['architecture'] == 'amd64' for i in identities.values()), 'wrong image architecture') + for image in images: + declared = json.loads(command(args, ['docker', 'image', 'inspect', '--format', '{{json .Config.Volumes}}', image]).stdout) or {} + covered = {MYSQL: {'/var/lib/mysql'}, REDIS: {'/data'}}.get(image, set()) + require(set(declared).issubset(covered), 'uncovered image volume declaration') + labels = json.loads(command(args, ['docker', 'image', 'inspect', '--format', '{{json .Config.Labels}}', SERVER]).stdout) + require(labels.get('org.opencontainers.image.revision') == SERVER_SOURCE, 'wrong PHP Server source') + fixture = args.work_dir / 'fixture' + export_consumer(fixture) + sources = {p.relative_to(fixture).as_posix(): native.sha256(p) + for p in fixture.rglob('*') if p.is_file()} + fetch = build(args, fixture, owner, 'cargo generate-lockfile && cargo fetch --locked', 'fetch') + (fixture / 'fetch.stdout').write_text(fetch.stdout) + package = verify_crate(fixture) + compiled = build(args, fixture, owner, 'rustc -V && cargo -V && cargo test --locked --offline --test conditions --no-run --message-format=json', + 'build', network='none') + (fixture / 'build.stdout').write_text(compiled.stdout) + (fixture / 'build.stderr').write_text(compiled.stderr) + toolchain = compiled.stdout.splitlines()[:2] + require(len(toolchain) == 2 and toolchain[0].startswith('rustc 1.86.0 ') + and toolchain[1].startswith('cargo 1.86.0 '), 'wrong actual Rust/Cargo toolchain') + artifacts = [json.loads(line) for line in compiled.stdout.splitlines() if line.startswith('{')] + executables = [p['executable'] for p in artifacts if p.get('reason') == 'compiler-artifact' + and p.get('target', {}).get('name') == 'conditions' and p.get('executable')] + require(len(executables) == 1 and executables[0].startswith('/work/target/debug/deps/conditions-'), 'wrong consumer executable') + binary = native.confined_child(fixture, executables[0].removeprefix('/work/')) + output = fixture / 'target/debug/condition-history' + shutil.copyfile(binary, output) + output.chmod(0o755) + binary_sha = native.sha256(output) + require(not native.run_objects('container', owner), 'build/runtime overlap') + cell = ConditionCell(types.SimpleNamespace(rust_image=RUST), CELL, args.work_dir, + {'executed_manifest': SERVER}, fixture, owner) + cell.deadline = min(int(time.time()) + 600, args.active_unix - 10) + outcome = cell.execute(args) + require(native.sha256(output) == binary_sha, 'consumer binary changed') + require(verify_crate(fixture) == package, 'public package changed') + require(all(native.sha256(fixture / path) == digest for path, digest in sources.items()), 'consumer source changed') + result.update(outcome='pass', package=package, binary_sha256=binary_sha, + input_sha256=INPUTS, consumer_source_sha256=sources, toolchain=toolchain, + registry=registry, images=identities, server_index=SERVER_INDEX, cell=outcome) + except BaseException as error: + result['error_type'] = type(error).__name__ + finally: + # Preserve only fenced Cell evidence during normal completion. Recovery + # purges interrupted cells, where the client no longer has secret bytes. + for kind in ('container', 'network'): + result['leftover_' + kind] = owner.best_effort('final-query', kind, native.run_objects, kind, owner) + result['cleanup_errors'] = owner.cleanup_errors + if owner.cleanup_errors or result['leftover_container'] != [] or result['leftover_network'] != []: + result['outcome'] = 'fail' + owner.best_effort('uncertain-recovery', CELL, recover, args.work_dir, owner.token) + data = json.dumps(result, indent=2).encode() + clean, leaked = native.redact_credentials(data, owner.secret_values) + if leaked: + result = {'outcome': 'fail', 'error_type': 'CredentialOutput'} + clean = json.dumps(result).encode() + owner.best_effort('credential-purge', CELL, recover, args.work_dir, owner.token) + (args.work_dir / 'summary.json').write_bytes(clean + b'\n') + # Cancel the backstop only after a direct zero-survivor readback. A + # failed query leaves it alive to perform the independent final sweep. + if native.run_objects('container', owner) == [] and native.run_objects('network', owner) == []: + watchdog.terminate() + watchdog.wait(timeout=10) + require(result['outcome'] == 'pass', 'acceptance failed; inspect bounded summary') + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=('run', 'recover')) + parser.add_argument('--work-dir', required=True, type=native.cli_path) + parser.add_argument('--run-token', required=True) + parser.add_argument('--package-commit') + parser.add_argument('--runner-commit') + parser.add_argument('--started-unix', type=int) + parser.add_argument('--active-unix', type=int) + parser.add_argument('--cleanup-unix', type=int) + parser.add_argument('--at', type=int) + args = parser.parse_args(argv) + if args.mode == 'recover': + if args.at is not None: + time.sleep(max(0, args.at - time.time())) + recover(args.work_dir, args.run_token) + else: + execute(args) + + +if __name__ == '__main__': + main() diff --git a/scripts/acceptance/condition-history/test_packet.py b/scripts/acceptance/condition-history/test_packet.py new file mode 100644 index 0000000..a1cda15 --- /dev/null +++ b/scripts/acceptance/condition-history/test_packet.py @@ -0,0 +1,158 @@ +"""Inert source controls. Every Docker/Cargo/process launch is stubbed.""" +import ast +import copy +import importlib.util +import json +import os +import pathlib +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('condition_packet', HERE / 'run.py') +packet = importlib.util.module_from_spec(spec) +spec.loader.exec_module(packet) + + +class Packet(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = pathlib.Path(self.tmp.name) + now = int(time.time()) + self.args = types.SimpleNamespace(work_dir=self.root / 'fresh', package_commit=packet.H, + runner_commit='a' * 40, run_token='0123456789abcdef', started_unix=now, + active_unix=now + 1740, cleanup_unix=now + 1800) + + def git(self, *args): + return {'HEAD': 'a' * 40, 'refs/tags/4.12.0^{commit}': packet.H}.get(args[-1], '') + + def test_preflight_refusals_precede_construction_and_launch(self): + cases = {'package_commit': 'b' * 40, 'runner_commit': 'not-a-commit', + 'run_token': 'foreign', 'active_unix': 0, 'started_unix': 0} + for field, value in cases.items(): + args = copy.copy(self.args) + setattr(args, field, value) + with self.subTest(field=field), mock.patch.object(packet, 'git', self.git), \ + mock.patch.object(packet, 'command') as command, \ + mock.patch.object(packet.subprocess, 'Popen') as launch, \ + self.assertRaises(SystemExit): + packet.execute(args) + command.assert_not_called() + launch.assert_not_called() + self.assertFalse(args.work_dir.exists()) + + def test_wrong_h_refuses_under_python_optimization(self): + result = subprocess.run([sys.executable, str(HERE / 'run.py'), 'run', '--work-dir', + str(self.args.work_dir), '--run-token', self.args.run_token, '--package-commit', 'b' * 40], + env=dict(os.environ, PYTHONOPTIMIZE='1'), capture_output=True, text=True, timeout=5) + self.assertNotEqual(result.returncode, 0) + self.assertIn('wrong fixed package source', result.stderr) + self.assertFalse(self.args.work_dir.exists()) + + def test_no_optimizable_assert_and_preflight_precedes_effects(self): + tree = ast.parse((HERE / 'run.py').read_text()) + self.assertFalse(any(isinstance(n, ast.Assert) for n in ast.walk(tree))) + execute = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'execute') + self.assertEqual(ast.unparse(execute.body[0]), 'preflight(args)') + + def test_dirty_head_tag_and_existing_work_refuse(self): + for response in ('dirty', 'wrong-head', 'wrong-tag', 'existing'): + def git(*args): + if response == 'dirty' and args[0] == 'status': + return ' M source' + if response == 'wrong-head' and args[-1] == 'HEAD': + return 'b' * 40 + if response == 'wrong-tag' and args[-1].startswith('refs/tags'): + return 'b' * 40 + return self.git(*args) + if response == 'existing': + self.args.work_dir.mkdir() + with self.subTest(response=response), mock.patch.object(packet, 'git', git), self.assertRaises(SystemExit): + packet.preflight(self.args) + + def test_exact_immutable_consumer_inputs_export_without_sdk_overlay(self): + fixture = self.root / 'fixture' + packet.export_consumer(fixture) + self.assertFalse((fixture / 'src').exists()) + self.assertEqual(packet.native.sha256(fixture / 'tests/official.rs'), + packet.INPUTS['tests/replay_regression_corpus.rs']) + for source, expected in packet.INPUTS.items(): + if source.endswith('.json'): + self.assertEqual(packet.native.sha256(fixture / source), expected) + manifest = packet.tomllib.loads((fixture / 'Cargo.toml').read_text()) + self.assertEqual(manifest['dependencies']['durable-workflow'], '=4.12.0') + self.assertNotIn('patch', manifest) + self.assertFalse((fixture / 'Cargo.lock').exists()) + self.assertFalse((fixture / '.cargo').exists()) + + def test_roles_keep_reviewed_bounds_and_credential_scopes(self): + owner = packet.native.Run(packet.PREFIX, token=self.args.run_token) + cell = packet.ConditionCell(types.SimpleNamespace(), packet.CELL, self.root, + {'executed_manifest': packet.SERVER}, self.root / 'fixture', owner) + calls = [] + def launch(_self, line, name, image, child, user, credentials): + calls.append((line, credentials)) + return name + def inspect(*args, **kwargs): + line, _ = calls[-1] + return types.SimpleNamespace(stdout=json.dumps({'id': 'synthetic', 'image': 'synthetic', + 'run': owner.token, 'owner': owner.prefix, 'pid': 1, 'mounts': 'tmpfs:/credentials;', + 'memory': int(line[line.index('--memory') + 1].removesuffix('m')) * 1024 * 1024, + 'nano_cpus': int(float(line[line.index('--cpus') + 1]) * 1_000_000_000)})) + with mock.patch.object(packet.native.Cell, 'launch', launch), mock.patch.object(packet.native, 'call', inspect): + for role, credentials in [('mysql', 'none'), ('redis', 'none'), ('bootstrap', 'server'), + ('runtime', 'both'), ('queue', 'server'), ('consumer', 'auth')]: + cell.launch(['docker', 'run', '--cpus', '1', '--memory', '1g'], cell.task + '-' + role, + packet.SERVER, ['original'], '1000:1000', credentials) + line, observed = calls[-1] + self.assertEqual(line[line.index('--cpus') + 1], packet.LIMITS[role][0]) + self.assertEqual(line[line.index('--memory') + 1], packet.LIMITS[role][1]) + self.assertEqual(observed, credentials) + cell.containers = ['held'] * 5 + with self.assertRaisesRegex(SystemExit, 'ceiling'): + cell.launch(['docker', 'run'], cell.task + '-consumer', packet.RUST, [], '1000:1000', 'auth') + + def report(self): + fixture = packet.native.call(['git', '-C', str(packet.REPO), 'show', packet.H + + ':tests/fixtures/replay-regressions/two-member-condition-group-server-reopen.json']).stdout + events = json.loads(fixture)['history'] + events.append({'event_type': 'WorkflowCompleted', 'payload': {}}) + return {'outcome': 'pass', 'result': 'both', 'initial': events[:2], 'reopened': events[:6], 'history': events} + + def test_canonical_group_shape_is_accepted(self): + result = packet.comparator.check(self.report()) + self.assertEqual(result['reopened_sequences'], [3, 4]) + + def test_group_comparator_rejects_lineage_order_timeout_and_false_health(self): + cases = [lambda r: r.update(result='one'), + lambda r: r['history'][4]['payload'].update(parallel_group_index=1), + lambda r: r['history'][4]['payload'].update(condition_wait_occurrence_id='new-call'), + lambda r: r['history'][4]['payload'].update(timeout_seconds=301), + lambda r: r['history'][0]['payload'].update(sequence=True), + lambda r: r['history'][4]['payload'].update(parallel_group_index=False), + lambda r: r['history'].insert(0, {'event_type': 'ReplayFailure'}), + lambda r: r['history'][3]['payload'].update(sequence=1), + lambda r: r['history'][-2]['payload'].update(signal_name='wrong')] + for change in cases: + report = self.report() + change(report) + with self.subTest(change=change), self.assertRaises((ValueError, KeyError, TypeError)): + packet.comparator.check(report) + + def test_foreign_recovery_record_refuses_before_docker(self): + self.args.work_dir.mkdir() + (self.args.work_dir / 'run.json').write_text(json.dumps({'owner': 'other', + 'token': self.args.run_token, 'cells': [packet.CELL]})) + with mock.patch.object(packet.native, 'recover_run') as recover, self.assertRaises(SystemExit): + packet.recover(self.args.work_dir, self.args.run_token) + recover.assert_not_called() + + +if __name__ == '__main__': + unittest.main() From 67a37f7f0e42ba63ee4e68cae3090a9ba73e1cc2 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:25:18 -0600 Subject: [PATCH 02/14] test: bind captured condition and timer member lineage --- .../acceptance/condition-history/README.md | 10 ++ .../condition-history/check_history.py | 135 ++++++++++++++---- .../fixture/tests/conditions.rs | 38 +++-- scripts/acceptance/condition-history/run.py | 1 + .../condition-history/test_packet.py | 78 ++++++++++ 5 files changed, 224 insertions(+), 38 deletions(-) diff --git a/scripts/acceptance/condition-history/README.md b/scripts/acceptance/condition-history/README.md index b882e4a..c263d7d 100644 --- a/scripts/acceptance/condition-history/README.md +++ b/scripts/acceptance/condition-history/README.md @@ -97,6 +97,16 @@ not an independent second HTTP client. After execution, a separate read-only own audit must confirm all matching containers, networks, volumes and processes absent. Publish a sanitized immutable result before acceptance or closure. +The comparator scans every opened wait, condition outcome and timer row, binding +the authored group, member key/fingerprint/occurrence, wait ID and integer timeout. +Additional reopens must keep that member lineage and cannot extend its recorded +timeout. Present timer terminals require a matching earlier schedule; a superseded +orphan's timer is distinct from the current tail. Current-tail timeouts contradict +this case's required satisfied result. H's minimal two-member fixture has no timer +rows, and its direct condition outcomes do not imply an invented timer schedule. +The live consumer also checks both public parallel results are `Satisfied` before +returning `both`. These are source assertions for review, not observed live results. + Inert source checks, which do not invoke Docker or Cargo: ```sh diff --git a/scripts/acceptance/condition-history/check_history.py b/scripts/acceptance/condition-history/check_history.py index c8c0436..28b6ef9 100644 --- a/scripts/acceptance/condition-history/check_history.py +++ b/scripts/acceptance/condition-history/check_history.py @@ -1,4 +1,15 @@ -"""Offline comparator for the newly captured two-member PHP history.""" +"""Offline comparator for the newly captured two-member PHP history. + +The declared identities follow unchanged H's two-member fixture. That minimal +fixture contains no timer rows, so a timer schedule is not mandatory. Every +timer row actually present must bind to an opened wait, and every timer terminal +must bind to its preceding schedule. Direct condition outcomes need no invented +timer row (H also supports that history shape). +""" + +MEMBERS = [('votes', 'sha256:votes-v1'), ('approvals', 'sha256:approvals-v1')] +GROUP = {'parallel_group_id': 'parallel-calls:1:2', 'parallel_group_kind': 'condition', + 'parallel_group_base_sequence': 1, 'parallel_group_size': 2} def require(condition, label): @@ -6,37 +17,110 @@ def require(condition, label): raise ValueError(label) +def member(payload): + require(isinstance(payload, dict), 'condition-payload') + require(all(type(payload.get(field)) is int for field in + ('sequence', 'parallel_group_index', 'parallel_group_base_sequence', 'parallel_group_size')), + 'typed-group-numbers') + index = payload['parallel_group_index'] + require(index in (0, 1) and all(payload.get(k) == v for k, v in GROUP.items()), 'group-identity') + key, fingerprint = MEMBERS[index] + require(payload.get('condition_key') == key + and payload.get('condition_definition_fingerprint') == fingerprint + and payload.get('condition_wait_occurrence_id') == 'rust:condition-wait:' + str(index), + 'authored-member-identity') + return index + + +def opened_wait(payload, waits): + sequence = payload.get('sequence') + require(type(sequence) is int and sequence in waits, 'outcome-before-open') + wait = waits[sequence] + require(payload.get('condition_wait_id') == wait['payload']['condition_wait_id'], 'wait-identity') + return wait + + +def open_wait(payload, waits, tails): + index = member(payload) + sequence, timeout = payload['sequence'], payload.get('timeout_seconds') + require(sequence == len(waits) + 1 and payload.get('condition_wait_id') == 'condition:' + str(sequence), + 'open-sequence-identity') + require(type(timeout) is int and 0 <= timeout <= 300, 'typed-timeout-bound') + if sequence <= 4: + require(index == (sequence - 1) % 2, 'initial-reopen-members') + previous = tails.get(index) + if previous is None: + require(sequence in (1, 2) and timeout == 300, 'original-timeout') + else: + require(timeout <= waits[previous]['payload']['timeout_seconds'], 'extended-timeout') + waits[sequence] = {'payload': payload, 'member': index, 'terminal': None} + tails[index] = sequence + + +def condition_outcome(kind, payload, waits, tails): + index = member(payload) + wait = opened_wait(payload, waits) + require(wait['member'] == index and wait['terminal'] is None, 'condition-terminal-identity') + if 'timeout_seconds' in payload: + require(type(payload['timeout_seconds']) is int + and payload['timeout_seconds'] == wait['payload']['timeout_seconds'], 'outcome-timeout') + if kind == 'ConditionWaitTimedOut': + require(tails[index] != payload['sequence'], 'current-tail-timeout') + wait['terminal'] = kind + + +def timer_row(kind, payload, waits, tails, timers): + require(isinstance(payload, dict), 'timer-payload') + wait = opened_wait(payload, waits) + timer_id = payload.get('timer_id') + require(isinstance(timer_id, str) and timer_id and payload.get('timer_kind') == 'condition_timeout', 'timer-identity') + require(type(payload.get('delay_seconds')) is int + and payload['delay_seconds'] == wait['payload']['timeout_seconds'], 'timer-delay') + identity = (payload['sequence'], payload['condition_wait_id'], payload['delay_seconds']) + if kind == 'TimerScheduled': + require(timer_id not in timers + and not any(t['identity'][0] == identity[0] for t in timers.values()), 'duplicate-timer-schedule') + timers[timer_id] = {'identity': identity, 'terminal': None} + else: + require(timer_id in timers and timers[timer_id]['identity'] == identity + and timers[timer_id]['terminal'] is None, 'unresolved-timer-terminal') + if kind == 'TimerFired': + require(tails[wait['member']] != payload['sequence'], 'current-tail-timer-fired') + timers[timer_id]['terminal'] = kind + + +def scan(events): + waits, tails, timers = {}, {}, {} + completed = False + for event in events: + require(isinstance(event, dict), 'history-row') + kind, payload = event.get('event_type'), event.get('payload') + if kind == 'WorkflowCompleted': + require(not completed, 'duplicate-completion') + completed = True + elif kind == 'ConditionWaitOpened': + require(not completed, 'open-after-completion') + open_wait(payload, waits, tails) + elif kind in ('ConditionWaitSatisfied', 'ConditionWaitTimedOut'): + require(not completed, 'condition-outcome-after-completion') + condition_outcome(kind, payload, waits, tails) + elif kind in ('TimerScheduled', 'TimerFired', 'TimerCancelled'): + require(not completed or kind == 'TimerCancelled', 'timer-after-completion') + timer_row(kind, payload, waits, tails, timers) + require(completed and len(waits) >= 4, 'completed-group-history') + return waits + + def check(report): require(report.get('outcome') == 'pass' and report.get('result') == 'both', 'typed-result') snapshots = [report.get(key) for key in ('initial', 'reopened', 'history')] require(all(isinstance(events, list) for events in snapshots), 'history-arrays') initial, reopened, final = snapshots require(reopened[:len(initial)] == initial and final[:len(reopened)] == reopened, 'append-only-history') - opens = [e for e in final if e.get('event_type') == 'ConditionWaitOpened'] - require(len(opens) >= 4, 'four-opens') - first = [e['payload'] for e in opens[:4]] - require(all(type(p.get(field)) is int for p in first - for field in ('sequence', 'parallel_group_index', 'parallel_group_base_sequence', 'parallel_group_size')), - 'typed-group-numbers') - require([p.get('sequence') for p in first] == [1, 2, 3, 4], 'open-sequences') - require([p.get('condition_key') for p in first] == ['votes', 'approvals', 'votes', 'approvals'], 'member-keys') - require([p.get('parallel_group_index') for p in first] == [0, 1, 0, 1], 'member-indices') - require(all(p.get('parallel_group_id') == 'parallel-calls:1:2' - and p.get('parallel_group_kind') == 'condition' - and p.get('parallel_group_base_sequence') == 1 - and p.get('parallel_group_size') == 2 for p in first), 'group-identity') - for index in (0, 1): - original, successor = first[index], first[index + 2] - for field in ('condition_wait_occurrence_id', 'condition_definition_fingerprint'): - require(isinstance(original.get(field), str) and original[field] == successor.get(field), field) - require(original.get('timeout_seconds') == 300 - and isinstance(successor.get('timeout_seconds'), int) - and not isinstance(successor['timeout_seconds'], bool) - and 0 <= successor['timeout_seconds'] <= 300, 'original-timeout-bound') - require(first[0]['condition_wait_occurrence_id'] != first[1]['condition_wait_occurrence_id'], 'distinct-members') + waits = scan(final) + require(sum(e.get('event_type') == 'ConditionWaitOpened' for e in initial) == 2, 'initial-two-members') types = [e.get('event_type') for e in final] require(not any(t in ('WorkflowFailed', 'ReplayFailure', 'WorkflowTimedOut') for t in types), 'no-replay-failure') - require(types.count('WorkflowCompleted') == 1, 'one-completion') signal_rows = [(i, e.get('payload', {}).get('signal_name')) for i, e in enumerate(final) if e.get('event_type') == 'SignalReceived'] require([name for _, name in signal_rows] == ['vote', 'vote', 'approve'], 'signal-order') @@ -45,5 +129,6 @@ def check(report): early_satisfied = [e.get('payload', {}).get('sequence') for e in reopened if e.get('event_type') == 'ConditionWaitSatisfied'] require(2 in early_satisfied and 1 not in early_satisfied, 'unresolved-first-member') + require(types.index('WorkflowCompleted') > signal_rows[-1][0], 'completion-after-approval') return {'outcome': 'pass', 'first_open_sequences': [1, 2], 'reopened_sequences': [3, 4], - 'captured_events': len(final), 'result': report['result']} + 'all_open_sequences': list(waits), 'captured_events': len(final), 'result': report['result']} diff --git a/scripts/acceptance/condition-history/fixture/tests/conditions.rs b/scripts/acceptance/condition-history/fixture/tests/conditions.rs index d52be5c..c64bafa 100644 --- a/scripts/acceptance/condition-history/fixture/tests/conditions.rs +++ b/scripts/acceptance/condition-history/fixture/tests/conditions.rs @@ -327,19 +327,31 @@ mod installed_conditions { worker.register_workflow("installed.two-member", |ctx, _| async move { let votes = ctx.clone(); let approvals = ctx.clone(); - ctx.parallel(vec![ - ParallelOperation::condition( - ConditionWaitOptions::new("votes", "sha256:votes-v1") - .timeout(Duration::from_secs(300)), - move || Ok(votes.signals("vote")?.len() >= 2), - ), - ParallelOperation::condition( - ConditionWaitOptions::new("approvals", "sha256:approvals-v1") - .timeout(Duration::from_secs(300)), - move || Ok(!approvals.signals("approve")?.is_empty()), - ), - ]) - .await?; + let outcomes = ctx + .parallel(vec![ + ParallelOperation::condition( + ConditionWaitOptions::new("votes", "sha256:votes-v1") + .timeout(Duration::from_secs(300)), + move || Ok(votes.signals("vote")?.len() >= 2), + ), + ParallelOperation::condition( + ConditionWaitOptions::new("approvals", "sha256:approvals-v1") + .timeout(Duration::from_secs(300)), + move || Ok(!approvals.signals("approve")?.is_empty()), + ), + ]) + .await?; + if !matches!( + outcomes.as_slice(), + [ + ParallelResult::Condition(ConditionWaitResult::Satisfied), + ParallelResult::Condition(ConditionWaitResult::Satisfied), + ] + ) { + return Err(Error::WorkerLoop( + "both live conditions must be satisfied".into(), + )); + } Ok(json!("both")) }); let (stop, stopped) = tokio::sync::oneshot::channel::<()>(); diff --git a/scripts/acceptance/condition-history/run.py b/scripts/acceptance/condition-history/run.py index cd7c39a..45a4a9f 100644 --- a/scripts/acceptance/condition-history/run.py +++ b/scripts/acceptance/condition-history/run.py @@ -238,6 +238,7 @@ def execute(self, args): def recover(root, token): + require(isinstance(token, str) and re.fullmatch('[0-9a-f]{16}', token), 'invalid recovery token') record = json.loads(native.confined_child(root, 'run.json').read_text()) require(record.get('owner') == PREFIX and record.get('token') == token and record.get('cells') == [CELL], 'wrong recovery ownership') diff --git a/scripts/acceptance/condition-history/test_packet.py b/scripts/acceptance/condition-history/test_packet.py index a1cda15..fcacc43 100644 --- a/scripts/acceptance/condition-history/test_packet.py +++ b/scripts/acceptance/condition-history/test_packet.py @@ -153,6 +153,84 @@ def test_foreign_recovery_record_refuses_before_docker(self): packet.recover(self.args.work_dir, self.args.run_token) recover.assert_not_called() + def test_malformed_recovery_token_refuses_before_reading_a_record(self): + with mock.patch.object(packet.native, 'confined_child') as read, self.assertRaises(SystemExit): + packet.recover(self.root / 'absent', 'malformed') + read.assert_not_called() + + def test_server_review_countercontrols(self): + cases = [] + report = self.report() + report['history'].insert(-1, {'event_type': 'TimerFired', 'payload': { + 'sequence': 3, 'condition_wait_id': 'foreign', 'timer_id': 'foreign', + 'timer_kind': 'condition_timeout', 'delay_seconds': 300}}) + cases.append(('foreign-timer', report)) + report = self.report() + report['history'].insert(-1, {'event_type': 'ConditionWaitTimedOut', 'payload': { + 'sequence': 999, 'condition_wait_id': 'foreign'}}) + cases.append(('foreign-timeout', report)) + report = self.report() + extra = copy.deepcopy(report['history'][4]) + extra['payload'].update(sequence=999, condition_wait_id='foreign', condition_key='foreign') + report['history'].insert(-1, extra) + cases.append(('foreign-fifth-open', report)) + report = self.report() + for row in report['history']: + if row['event_type'].startswith('ConditionWait'): + row['payload']['condition_definition_fingerprint'] = 'sha256:foreign' + cases.append(('consistently-wrong-fingerprint', report)) + report = self.report() + report['history'][0]['payload']['timeout_seconds'] = 300.0 + cases.append(('float-original-timeout', report)) + for label, report in cases: + with self.subTest(label=label), self.assertRaises(ValueError): + packet.comparator.check(report) + + def snapshots(self, events): + signals = [i for i, event in enumerate(events) if event['event_type'] == 'SignalReceived'] + return {'outcome': 'pass', 'result': 'both', 'initial': events[:signals[0]], + 'reopened': events[:signals[1]], 'history': events} + + def test_additional_reopens_keep_the_same_declared_members(self): + events = self.report()['history'] + extra = copy.deepcopy(events[4:6]) + for sequence, event in zip((5, 6), extra): + event['payload'].update(sequence=sequence, condition_wait_id='condition:' + str(sequence)) + events[6:6] = extra + report = self.snapshots(events) + self.assertEqual(packet.comparator.check(report)['all_open_sequences'], [1, 2, 3, 4, 5, 6]) + + def timer(self, kind, sequence): + return {'event_type': kind, 'payload': {'sequence': sequence, + 'condition_wait_id': 'condition:' + str(sequence), 'timer_id': 'timer:' + str(sequence), + 'timer_kind': 'condition_timeout', 'delay_seconds': 300}} + + def test_correlated_superseded_orphan_timer_is_distinct_from_the_current_tail(self): + events = self.report()['history'] + orphan = copy.deepcopy(events[0]) + orphan['event_type'] = 'ConditionWaitTimedOut' + events.insert(1, self.timer('TimerScheduled', 1)) + events[-3:-3] = [self.timer('TimerFired', 1), orphan] + self.assertEqual(packet.comparator.check(self.snapshots(events))['outcome'], 'pass') + + def test_timer_terminals_require_prior_schedule_and_row_ordered_supersession(self): + for label in ('missing-schedule', 'current-tail', 'before-reopen', 'duplicate-fire', 'wrong-delay'): + events = self.report()['history'] + if label == 'current-tail': + events.insert(5, self.timer('TimerScheduled', 3)) + events.insert(-1, self.timer('TimerFired', 3)) + else: + if label != 'missing-schedule': + events.insert(1, self.timer('TimerScheduled', 1)) + fired = self.timer('TimerFired', 1) + if label == 'wrong-delay': + fired['payload']['delay_seconds'] = 299 + events.insert(2 if label == 'before-reopen' else -1, fired) + if label == 'duplicate-fire': + events.insert(-1, copy.deepcopy(fired)) + with self.subTest(label=label), self.assertRaises(ValueError): + packet.comparator.check(self.snapshots(events)) + if __name__ == '__main__': unittest.main() From 8c232ffe7e8a60be10862c5949a0fd45135922af Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:03:19 -0600 Subject: [PATCH 03/14] WIP saved when sdk-rust stopped on lab3 --- .../acceptance/condition-history/README.md | 160 +++++--- .../acceptance/condition-history/engine.py | 323 +++++++++++++++ .../acceptance/condition-history/recover.py | 342 ++++++++++++++++ .../acceptance/condition-history/resources.py | 216 ++++++++++ .../condition-history/role-supervisor.sh | 41 ++ scripts/acceptance/condition-history/run.py | 378 +----------------- .../acceptance/condition-history/subject.py | 237 +++++++++++ .../condition-history/test_audit.py | 105 +++++ .../condition-history/test_packet.py | 143 ++----- .../condition-history/test_resources.py | 170 ++++++++ .../condition-history/test_subject.py | 241 +++++++++++ 11 files changed, 1849 insertions(+), 507 deletions(-) create mode 100644 scripts/acceptance/condition-history/engine.py create mode 100644 scripts/acceptance/condition-history/recover.py create mode 100644 scripts/acceptance/condition-history/resources.py create mode 100644 scripts/acceptance/condition-history/role-supervisor.sh create mode 100644 scripts/acceptance/condition-history/subject.py create mode 100644 scripts/acceptance/condition-history/test_audit.py create mode 100644 scripts/acceptance/condition-history/test_resources.py create mode 100644 scripts/acceptance/condition-history/test_subject.py diff --git a/scripts/acceptance/condition-history/README.md b/scripts/acceptance/condition-history/README.md index c263d7d..d3e73fe 100644 --- a/scripts/acceptance/condition-history/README.md +++ b/scripts/acceptance/condition-history/README.md @@ -35,67 +35,123 @@ All executable image references select immutable `linux/amd64` manifests: | MySQL | `mysql@sha256:80f4933e3835f9dc4d35a28ec500d7986cb4414e6c6821c5461239cb7beb8995` | | Redis | `redis@sha256:ca0acbb137c1dc3339c8b147a58fd6f42775d4599327b50e7b116c23de501af2` | -The Server index is `sha256:3848d5d5cd1b23bd2881e3fdecea64cc44b72e6fb812e577c2db6d0872d065dd`, -source `fa3fc2f2f6b8935ea4a6eea47f6a9221e2adafff`. The MySQL and Redis manifests -were resolved from the existing installed-native pins using digest-verified public -registry index reads. Their indices remain -`sha256:6ea90827b1100f8f2ae306a539f86d2c264a26ed435a2a9f75551dd5c3aeb242` -and `sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499`. - -The build uses two jobs and 3 GiB, with a fresh Cargo home, lock, downloaded crate -and test binary. The public sparse index, lock, archive and VCS commit must agree. -Only dependency resolution/fetch has network access. Compilation is offline. -There is no SDK source replacement or reuse of a prior acceptance binary/cache. +Provenance is limited to those executed child manifests and their effective +`linux/amd64` content readbacks. The Server source label must equal +`fa3fc2f2f6b8935ea4a6eea47f6a9221e2adafff`. This packet makes no registry-index +membership claim. + +The closed subject manifest binds Git R, H and every executable member's Git +blob, SHA-256, mode and size. `run.py` loads the subject directly from Git R. +The subject captures the engine, comparator, Rust fixture, inherited helper and +supervisor, new role supervisor and recovery program from allowlisted Git blobs. +It materializes a private read-only snapshot and refuses extra, missing, linked +or nonregular members. Imports, fixture exports and recovery launch use the +captured bytes. Python runs with `-I -S`; no checkout file is imported or reopened +as executable code after preflight. The inherited launch methods are overridden +by the literal role launcher in this packet. + +The build has two sequential phases with two Cargo jobs, 2 CPU, 3 GiB memory, +equal total memory-swap, 1024 PIDs, non-root user, read-only root, all capabilities +dropped and no-new-privileges. Cargo uses a fresh home, lock, public archive and +binary. The public sparse index, lock, archive and VCS commit must agree. Fetch +alone has the default network. Compilation is offline, with source mounted +read-only and only Cargo cache and target output writable. No prior acceptance +binary, lock or registry cache is reused. Runtime starts after all build containers are absent. Loopback replay runs alone. -The PHP case has at most five roles: MySQL 1 CPU/1.25 GiB, Redis 0.25 CPU/128 MiB, -HTTP 0.75 CPU/768 MiB, queue 0.5 CPU/512 MiB, and a combined Worker/driver -0.5 CPU/768 MiB. Bootstrap is removed before HTTP and queue startup. Total runtime -caps are 3 CPU/3.375 GiB. There is no repair or proxy role. Each role retains the -accepted installed-native supervisor, private stdin-to-tmpfs credential delivery, -dual ownership labels, evidence fence and cleanup. Database/cache data is tmpfs; -image volume declarations must be fully covered before any runtime starts. -Docker uses an empty registry configuration and the existing local Unix daemon. -No host credential, Docker socket or published port is mounted into a container. +The PHP case has at most five roles: + +| Role | CPU | Memory and equal total swap | PIDs | User | +| --- | ---: | ---: | ---: | --- | +| MySQL | 1 | 1280 MiB | 512 | 0:0, image drops to service user | +| Redis | 0.25 | 128 MiB | 256 | 0:0, image drops to service user | +| HTTP | 0.75 | 768 MiB | 256 | 0:0, image starts its service processes | +| Queue | 0.5 | 512 MiB | 256 | 1000:1000 | +| Worker and driver | 0.5 | 768 MiB | 256 | Non-root owner UID:GID | + +Bootstrap uses the HTTP CPU/memory/PID profile and is removed before HTTP and +queue startup. Total runtime caps are 3 CPU/3.375 GiB. `resources.PROFILES` +literally defines capability additions for each service; all roles drop ALL +before those additions. Worker has a read-only root and read-only fixture/binary +mounts, with only task state writable. Database/cache data and PHP cached +configuration use bounded tmpfs. Image volume declarations must be fully covered. + +Each role is created and started while its inline captured supervisor waits for +a private ready marker. Before releasing that gate, the owner independently reads +back image/content/platform, resource and swap/PID caps, user, root mode, +capabilities, init/security settings, exact command/deadline, environment, +network/internal ownership and mount destinations/types/access. Tokens and the +Server key arrive through private stdin into role tmpfs only after readback. +No host credential, Docker socket, published port or host network is admitted. +The owner uses a fixed anonymous Docker configuration and an explicit local Unix +daemon. Before work creation or registry requests, read-only preflight refuses +any existing owner/run container, network or volume and every colliding exact name. +An uncertain query is a refusal. One 30-minute absolute owner window includes setup, fetch, compilation, execution -and cleanup. Active work ends 60 seconds before the final ceiling. Each role in -the one Cell shares a maximum 600-second deadline, followed by TERM/KILL. -A detached owner recovery process is armed before pulls or build. It removes only -the exact owner/run labels and purges the interrupted Cell tree, since credential -values intentionally never enter `run.json`. It is canceled only after direct -zero-survivor readbacks. A failed attempt is retained as failed; no edit, retry, -fallback or extension is part of this command. - -After the exact packet is reviewed and admitted, substitute the reviewed runner -commit R, a fresh absolute start S, an unused private work directory W and a -16-hex owner token T in this one command: +and cleanup. Active work ends 60 seconds before the final ceiling. The Cell's +roles share one maximum 600-second deadline plus a 10-second TERM/KILL grace. +`run.json` is an exclusive owner-only regular single-link record binding R, H, +manifest and recovery digests, token, local daemon, root, UID/GID, isolated owner +PID/group/session epoch and all three deadlines. Its digest is captured outside +the work root in the private launch handoff. + +Before any pull, the owner starts a detached recovery process from captured +inline bytes and requires its exact readiness receipt. At the active cutoff it +uses its captured complete authority to stop the exact owner and remove only +matching immutable container/network IDs. A later source/record drift fails the +result but cannot replace the already-captured cleanup authority. Foreign process +epochs, labels or names are refused. Interrupted Cell evidence is purged because +credential values are deliberately absent from the recovery record. Cleanup is +bounded by the final ceiling. No edit, retry, fallback or extension is included. + +After source review and a separate exact execution admission, use the reviewed +inline collector below. `R`, `M`, `CHECKOUT`, fresh `W`, token `T`, local daemon +`D`, and captured `S`/`ACTIVE`/`CLEANUP` belong in the private proposal. `ACTIVE` +is `S + 1740` and `CLEANUP` is `S + 1800`. Do not substitute a mutable script path +for the captured entry point. ```sh -python3 scripts/acceptance/condition-history/run.py run \ +python3 -I -S -c ' +import pathlib, re, subprocess, sys +checkout, revision = sys.argv[1:3] +if not re.fullmatch("[0-9a-f]{40}", revision): + raise SystemExit("invalid reviewed runner") +entry = subprocess.run(["git", "--no-replace-objects", "-C", checkout, "show", + revision + ":scripts/acceptance/condition-history/run.py"], + check=True, capture_output=True, timeout=20).stdout.decode() +raise SystemExit(subprocess.run([sys.executable, "-I", "-S", "-c", entry, + "run", "--checkout", checkout, "--runner-commit", revision, *sys.argv[3:]], + start_new_session=True).returncode) +' CHECKOUT R --manifest-sha256 M \ --package-commit 4b2440a5618ccc1c98960ea2e1e62127042c3590 \ - --runner-commit R --work-dir W --run-token T \ + --work-dir W --run-token T --docker-endpoint D \ --started-unix S --active-unix ACTIVE --cleanup-unix CLEANUP ``` -`ACTIVE` must equal `S + 1740`; `CLEANUP` must equal `S + 1800`. The runner refuses -wrong H, a dirty/wrong R, an existing work directory or stale/malformed deadlines -before harness command construction or launch. Host/capacity facts and actual -private paths belong in task evidence, not public GitHub prose. - -If the client dies, the same reviewed source supports immediate standalone cleanup: - -```sh -python3 scripts/acceptance/condition-history/run.py recover --work-dir W --run-token T -``` - -The summary retains source/input/lock/package/binary identities, actual toolchain, -role ownership/resource readbacks, typed replay outputs, fresh raw live history and -an offline lineage comparator. The comparator has adversarial lineage/order/timer -controls. Its fetch transport is the public SDK raw control-plane method; it is -not an independent second HTTP client. After execution, a separate read-only owner -audit must confirm all matching containers, networks, volumes and processes absent. -Publish a sanitized immutable result before acceptance or closure. +The collector preserves the exact Git blob's trailing newline and starts a new +owner session. A `manifest` invocation of the same captured entry reads Git only +and prints the deterministic manifest and its digest. It does not contact Docker +or registries. The run refuses wrong H, dirty/wrong R, wrong M, existing work, +stale deadlines or a changed entry before importing the engine or creating roles. +Host aliases, live capacity samples and private paths stay out of public records. + +For immediate recovery or the mandatory independent final audit, capture +`recover.py` directly from the reviewed Git R and invoke those bytes with +`python3 -I -S -c`. Supply `recover` or `audit` and all of `--work-dir`, +`--record-sha256`, `--manifest-sha256`, `--runner-commit`, `--package-commit`, +`--run-token`. Audit also requires the externally captured `--summary-sha256`. +Standalone recovery does not depend on a mutable checkout entry point. + +The owner writes `summary.json` with `O_EXCL|O_NOFOLLOW`, mode 0600, and returns +its immutable SHA-256. It records package/source/lock/binary inputs, exact role +readbacks and fresh typed replay/live results. The fetch transport is the public +SDK raw control-plane method, not an independent second HTTP client. Owner checks +remain `pending_independent_audit`. Only after owner exit can a separately invoked +captured audit bind that summary digest, verify the original record/snapshot and +recovery-process receipt, and confirm every matching container, network, volume, +owner and recovery process group absent. It writes its own exclusive handoff. +Sanitize and retain both immutable results before any acceptance or closure. The comparator scans every opened wait, condition outcome and timer row, binding the authored group, member key/fingerprint/occurrence, wait ID and integer timeout. @@ -110,7 +166,7 @@ returning `both`. These are source assertions for review, not observed live resu Inert source checks, which do not invoke Docker or Cargo: ```sh -python3 -m unittest discover -s scripts/acceptance/condition-history -p test_packet.py +python3 -m unittest discover -s scripts/acceptance/condition-history -p 'test_*.py' rustfmt --edition 2021 --check scripts/acceptance/condition-history/fixture/tests/conditions.rs ``` diff --git a/scripts/acceptance/condition-history/engine.py b/scripts/acceptance/condition-history/engine.py new file mode 100644 index 0000000..70a71d8 --- /dev/null +++ b/scripts/acceptance/condition-history/engine.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""One explicitly admitted public-package condition-history acceptance. + +Source preparation and stubbed tests do not authorize invoking this runner. +See README.md for its fixed tuple, command and remaining review gate. +""" +import json +import os +import pathlib +import shutil +import subprocess +import sys +import time +import tomllib +import types +import urllib.request +import sealed_subject as subject + +HERE = subject.CURRENT.root / 'scripts/acceptance/condition-history' +REPO = pathlib.Path(subject.ARGS.checkout) +native = subject.CURRENT.module('native') +comparator = subject.CURRENT.module('comparator') +resources = subject.CURRENT.module('resources') +recovery = subject.CURRENT.module('recovery') + +H = '4b2440a5618ccc1c98960ea2e1e62127042c3590' +CRATE_SHA = '620a8287af73c363dac61f0a91f20971a39f3b8cdc9513fbea85cdbd95a74ba4' +RUST = 'rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081' +SERVER = 'ghcr.io/durable-workflow/server@sha256:980fe0a052089ced34ec3fcd2be2ea4e77436861d8fb537fb6654cb652a53e07' +SERVER_SOURCE = 'fa3fc2f2f6b8935ea4a6eea47f6a9221e2adafff' +MYSQL = 'mysql@sha256:80f4933e3835f9dc4d35a28ec500d7986cb4414e6c6821c5461239cb7beb8995' +REDIS = 'redis@sha256:ca0acbb137c1dc3339c8b147a58fd6f42775d4599327b50e7b116c23de501af2' +INPUTS = { + 'tests/replay_regression_corpus.rs': 'b54be69c5b82ec6ad32bd2871c3451f182185fa01ca3316a1c3aee935c48c9cc', + 'tests/fixtures/replay-regressions/two-member-condition-group-server-reopen.json': + '815526c761ce314960335b0f7cafaa0aa60fa519ddd08252451a7a5ebdca821d', + 'tests/fixtures/replay-regressions/timer-separated-condition-group-server-reopen.json': + 'f86ecb0cc57634a3eb4c87ffbd4fa55555b82c2794a16cd7b4ce9d33e009d0f6', + 'tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json': + '50a630596b5d85a392a33cc595fe58a513f59f6ebdbba928fa22be4ea0fb9075', + 'tests/fixtures/replay-regressions/condition-orphaned-duplicate-timeout-keeps-wait.json': + '012790d1fc23fe11ef9208d3a8243dc282d160903f7df98396c77e1f8fd898f7', +} +CELL = 'condition-history' +PREFIX = 'sdk-rust-condition' +LIMITS = {'mysql': ('1', '1280m'), 'redis': ('0.25', '128m'), + 'bootstrap': ('0.75', '768m'), 'runtime': ('0.75', '768m'), + 'queue': ('0.5', '512m'), 'consumer': ('0.5', '768m')} + + +def require(condition, label): + if not condition: + raise SystemExit(label) + + +def command(args, argv, **kwargs): + subject.CURRENT.verify() + return resources.call(args, argv, **kwargs) + + +def export_consumer(fixture): + fixture.mkdir(mode=0o700) + subject.CURRENT.export(subject.PREFIX + 'fixture/', fixture) + for source, expected in INPUTS.items(): + raw = command(subject.ARGS, ['git', '--no-replace-objects', '-C', str(REPO), + 'show', H + ':' + source], text=False).stdout + require(subject.digest(raw) == expected, 'fixed history input differs') + target = fixture / ('tests/official.rs' if source == 'tests/replay_regression_corpus.rs' else source) + target.parent.mkdir(parents=True, exist_ok=True) + subject.exclusive(target, raw) + + +def build(args, fixture, controller, script, phase, network=None): + user = '%d:%d' % (os.getuid(), os.getgid()) + spec = controller.profile(phase, RUST, ['sh', '-ec', script], user, 'none', fixture, + env={'CARGO_HOME': '/work/.cargo', 'CARGO_BUILD_JOBS': '2', + 'CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS': 'fallback'}, + deadline=args.active_unix - 10) + name = controller.start(spec) + try: + code = command(args, ['docker', 'wait', name], timeout=max(1, args.active_unix - time.time())).stdout.strip() + logs = command(args, ['docker', 'logs', '--tail', '2000', name], check=False) + require(code == '0', 'bounded build phase failed') + return logs + finally: + object_id = recovery.owned(controller.record, 'container', name) + if object_id: + recovery.docker(controller.record, ['rm', '-fv', object_id]) + + +def verify_crate(fixture): + lock = tomllib.loads((fixture / 'Cargo.lock').read_text()) + packages = [p for p in lock['package'] if p['name'] == 'durable-workflow'] + require(len(packages) == 1 and packages[0]['version'] == '4.12.0' + and packages[0]['source'] == native.CRATES_IO_SOURCE + and packages[0]['checksum'] == CRATE_SHA, 'wrong public package lock') + archives = list((fixture / '.cargo/registry/cache').glob('*/durable-workflow-4.12.0.crate')) + require(len(archives) == 1 and native.sha256(archives[0]) == CRATE_SHA, 'wrong downloaded archive') + with native.tarfile.open(archives[0]) as archive: + vcs = json.load(archive.extractfile('durable-workflow-4.12.0/.cargo_vcs_info.json')) + require(vcs['git']['sha1'] == H and not vcs['git'].get('dirty'), 'wrong public package VCS') + require(not (fixture / '.cargo/config.toml').exists(), 'registry replacement is forbidden') + return {'version': '4.12.0', 'source': H, 'archive_sha256': CRATE_SHA, + 'lock_sha256': native.sha256(fixture / 'Cargo.lock')} + + +def registry_identity(): + request = urllib.request.Request('https://index.crates.io/du/ra/durable-workflow', + headers={'User-Agent': 'durable-workflow-condition-history-acceptance'}) + with urllib.request.urlopen(request, timeout=30) as response: + versions = [json.loads(line) for line in response.read(8 * 1024 * 1024).splitlines()] + exact = [version for version in versions if version.get('vers') == '4.12.0'] + require(len(exact) == 1 and exact[0].get('cksum') == CRATE_SHA + and exact[0].get('yanked') is False, 'public registry identity refused') + return {'version': '4.12.0', 'sha256': CRATE_SHA, 'yanked': False} + + +class ConditionCell(native.Cell): + def __init__(self, *args): + super().__init__(*args) + self.receipts = [] + + def launch(self, line, name, image, child, user, credentials): + role = name.removeprefix(self.task + '-') + require(role in LIMITS and len(self.containers) < 5, 'unknown role or runtime ceiling') + env = dict(line[i + 1].split('=', 1) for i, value in enumerate(line) if value == '-e') + alias = line[line.index('--network-alias') + 1] if '--network-alias' in line else None + spec = self.controller.profile(role, image, child, user, credentials, self.fixture, self.state, + env=env, alias=alias, deadline=self.deadline) + self.containers.append(name) + return self.controller.start(spec, + self.token if credentials in ('auth', 'both') else '', + self.server_key if credentials in ('server', 'both') else '') + + def remove_docker_objects(self): + errors_before = len(self.harness_run.cleanup_errors) + self.harness_run.best_effort('logs-dir', CELL, lambda: self.cell_path('logs').mkdir(exist_ok=True)) + for name in self.containers: + try: + object_id = recovery.owned(self.controller.record, 'container', name) + if object_id: + self.harness_run.best_effort('container-logs', name, self.save_logs, self.dir / 'logs', name, object_id) + except BaseException: + self.harness_run.cleanup_errors.append({'step': 'ownership', 'object': name, 'error': 'Refused'}) + result = recovery.cleanup(self.controller.record) + if not result['pass']: + self.harness_run.cleanup_errors.append({'step': 'cleanup', 'object': CELL, 'error': 'Uncertain'}) + return result['pass'] and not any(e['step'] == 'ownership' for e in self.harness_run.cleanup_errors[errors_before:]) + + def start_php(self, token): + env = {'APP_ENV': 'testing', 'APP_DEBUG': 'false', 'DB_CONNECTION': 'mysql', + 'DB_HOST': 'mysql', 'DB_DATABASE': 'durable_workflow', 'DB_USERNAME': 'workflow', + 'DB_PASSWORD': 'workflow', 'REDIS_HOST': 'redis', 'QUEUE_CONNECTION': 'redis', + 'CACHE_STORE': 'redis', 'DW_AUTH_DRIVER': 'token', 'DW_WORKER_PROTOCOL_VERSION': '1.20'} + mysql = self.plain('mysql', MYSQL, [], alias='mysql', tmpfs='/var/lib/mysql', pids=512, + caps=('CHOWN', 'DAC_OVERRIDE', 'SETGID', 'SETUID'), + env={'MYSQL_DATABASE': 'durable_workflow', 'MYSQL_USER': 'workflow', + 'MYSQL_PASSWORD': 'workflow', 'MYSQL_ROOT_PASSWORD': 'root'}) + self.plain('redis', REDIS, [], alias='redis', tmpfs='/data', caps=('CHOWN', 'SETGID', 'SETUID')) + until = min(time.time() + 120, self.deadline - 15) + while native.call(['docker', 'exec', mysql, 'mysqladmin', 'ping', '-h', 'localhost', '-uworkflow', '-pworkflow'], + check=False).returncode: + require(time.time() < until, 'mysql readiness deadline') + time.sleep(1) + bootstrap = self.plain('bootstrap', SERVER, ['server-bootstrap'], env=env, + caps=('DAC_OVERRIDE',), credentials='server') + require(native.call(['docker', 'wait', bootstrap], timeout=max(1, self.deadline - time.time())).stdout.strip() == '0', + 'PHP bootstrap failed') + object_id = recovery.owned(self.controller.record, 'container', bootstrap) + require(object_id, 'bootstrap ownership changed') + recovery.docker(self.controller.record, ['rm', '-fv', object_id]) + self.containers.remove(bootstrap) + self.plain('runtime', SERVER, [], env=env, alias='qualification-origin', credentials='both', + caps=('CHOWN', 'DAC_OVERRIDE', 'KILL', 'SETGID', 'SETUID')) + self.plain('queue', SERVER, ['php', 'artisan', 'queue:work', '--sleep=1', '--tries=3', '--max-time=600'], + env=env, user='1000:1000', credentials='server') + + def execute(self, args): + outcome = {'outcome': 'fail', 'deadline_unix': self.deadline} + try: + self.cell_path('state').mkdir(parents=True) + self.dir.chmod(0o777) + self.state.chmod(0o777) + network = ['docker', 'network', 'create', '--internal'] + for key, value in self.controller.labels('network').items(): + network += ['--label', key + '=' + value] + command(args, network + [self.task]) + # No service roles exist during loopback replay; then exactly four + # service roles plus one combined driver/Worker. No proxy/repair. + replay = self.start('consumer', RUST, ['/fixture/condition-history', '--exact', + 'installed_conditions::installed_history_replay', '--nocapture']) + require(command(args, ['docker', 'wait', replay], timeout=300).stdout.strip() == '0', 'installed replay failed') + object_id = recovery.owned(self.controller.record, 'container', replay) + require(object_id, 'replay ownership changed') + recovery.docker(self.controller.record, ['rm', '-fv', object_id]) + self.containers.remove(replay) + self.start_php(self.token) + consumer = self.start('consumer', RUST, ['/fixture/condition-history', '--exact', + 'installed_conditions::installed_live_group', '--ignored', '--nocapture'], credentials='auth') + require(command(args, ['docker', 'wait', consumer], timeout=180).stdout.strip() == '0', 'live group failed') + finally: + stopped = self.remove_docker_objects() + try: + require(stopped, 'writer cleanup uncertain') + self.fence_artifacts() + require(not self.harness_run.cleanup_errors, 'evidence fence failed') + replay = json.loads(self.artifact_bytes(self.state / 'replay.json')) + live = json.loads(self.artifact_bytes(self.state / 'live.json')) + require(replay.get('outcome') == 'pass' and len(replay.get('cases', {})) == 9, 'replay report incomplete') + outcome.update(outcome='pass', replay=replay, live=live, history_check=comparator.check(live), + role_receipts=self.controller.receipts) + except BaseException: + outcome['outcome'] = 'fail' + self.harness_run.best_effort('evidence-purge', CELL, self.purge_evidence) + raise + return outcome + + +def execute(args): + # The captured bootstrap has already verified R/H/manifest and refused all + # collisions before creating this root or contacting a registry. + record = recovery.validate_record(args.work_dir, subject.RECORD_SHA, args.runner_commit, + subject.CURRENT.sha256, subject.RECORD['recovery_sha256'], args.run_token) + owner = native.Run(PREFIX, token=args.run_token) + owner.base = PREFIX + '-' + args.run_token + controller = resources.Docker(args, subject.CURRENT, record) + native.call = lambda line, **kwargs: command(args, line, **kwargs) + program = subject.CURRENT.read(subject.PREFIX + 'recover.py').decode() + require(subject.digest(program.encode()) == record['recovery_sha256'], 'captured recovery source mismatch') + recovery_argv = [sys.executable, '-I', '-S', '-c', program, 'recover', '--work-dir', args.work_dir, + '--record-sha256', subject.RECORD_SHA, '--manifest-sha256', subject.CURRENT.sha256, + '--runner-commit', args.runner_commit, '--package-commit', H, '--run-token', args.run_token, + '--at', str(args.active_unix)] + watchdog = subprocess.Popen(recovery_argv, start_new_session=True, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=resources.environment(args)) + process = recovery.process_identity(watchdog.pid) + require(process is not None, 'recovery did not start') + recovery_process_sha = subject.exclusive(pathlib.Path(args.work_dir) / 'recovery-process.json', subject.canonical({ + 'source_sha256': record['recovery_sha256'], 'process': process})) + try: + ready_path = pathlib.Path(args.work_dir) / 'recovery-ready.json' + until = min(time.time() + 5, args.active_unix - 10) + while not ready_path.exists(): + require(watchdog.poll() is None and time.time() < until, 'captured recovery not ready') + time.sleep(0.05) + ready = json.loads(recovery.read(ready_path)) + require(ready == {'record_sha256': subject.RECORD_SHA, 'manifest_sha256': subject.CURRENT.sha256, + 'recovery_sha256': record['recovery_sha256'], 'process': process}, 'wrong recovery-ready authority') + except BaseException: + if watchdog.poll() is None: + watchdog.terminate();watchdog.wait(timeout=5) + raise + result = {'outcome': 'fail', 'checks_outcome': 'fail', 'record_sha256': subject.RECORD_SHA, + 'runner_commit': args.runner_commit, 'package_commit': H, 'manifest_sha256': subject.CURRENT.sha256, + 'recovery_process_sha256': recovery_process_sha} + try: + registry = registry_identity() + images = [RUST, SERVER, MYSQL, REDIS] + for image in images: + command(args, ['docker', 'pull', '--platform', 'linux/amd64', image], timeout=600) + identities = {image: {'kind': 'executed-child-manifest', 'reference': image, + 'content_sha256': controller.image(image)['sha256'], 'os': 'linux', 'architecture': 'amd64'} for image in images} + labels = controller.image(SERVER)['raw']['Config'].get('Labels') or {} + require(labels.get('org.opencontainers.image.revision') == SERVER_SOURCE, 'wrong PHP Server source') + fixture = pathlib.Path(args.work_dir) / 'fixture' + export_consumer(fixture) + sources = {p.relative_to(fixture).as_posix(): native.sha256(p) for p in fixture.rglob('*') if p.is_file()} + fetch = build(args, fixture, controller, 'cargo generate-lockfile && cargo fetch --locked', 'fetch') + subject.exclusive(fixture / 'fetch.stdout', fetch.stdout.encode()) + require(all(native.sha256(fixture / path) == sha for path, sha in sources.items()), 'fetch changed consumer source') + package = verify_crate(fixture) + (fixture / 'target').mkdir(exist_ok=True) + compiled = build(args, fixture, controller, + 'rustc -V && cargo -V && cargo test --locked --offline --test conditions --no-run --message-format=json', 'build') + subject.exclusive(fixture / 'build.stdout', compiled.stdout.encode()) + subject.exclusive(fixture / 'build.stderr', compiled.stderr.encode()) + toolchain = compiled.stdout.splitlines()[:2] + require(len(toolchain) == 2 and toolchain[0].startswith('rustc 1.86.0 ') + and toolchain[1].startswith('cargo 1.86.0 '), 'wrong actual Rust/Cargo toolchain') + artifacts = [json.loads(line) for line in compiled.stdout.splitlines() if line.startswith('{')] + executables = [p['executable'] for p in artifacts if p.get('reason') == 'compiler-artifact' + and p.get('target', {}).get('name') == 'conditions' and p.get('executable')] + require(len(executables) == 1 and executables[0].startswith('/work/target/debug/deps/conditions-'), 'wrong consumer executable') + binary = native.confined_child(fixture, executables[0].removeprefix('/work/')) + output = fixture / 'target/debug/condition-history' + # Exclusive binary handoff, never an overwrite of a pre-existing name. + subject.exclusive(output, subject.regular(binary, limit=256 * 1024 * 1024), 0o500) + binary_sha = native.sha256(output) + require(not recovery.objects(record, 'container'), 'build/runtime overlap') + cell = ConditionCell(types.SimpleNamespace(rust_image=RUST), CELL, pathlib.Path(args.work_dir), + {'executed_manifest': SERVER}, fixture, owner) + cell.controller = controller + cell.deadline = min(int(time.time()) + 600, args.active_unix - 10) + outcome = cell.execute(args) + require(native.sha256(output) == binary_sha and verify_crate(fixture) == package, 'consumer/package changed') + require(all(native.sha256(fixture / path) == sha for path, sha in sources.items()), 'consumer source changed') + result.update(outcome='pending_independent_audit', checks_outcome='pass', package=package, + binary_sha256=binary_sha, input_sha256=INPUTS, consumer_source_sha256=sources, + toolchain=toolchain, registry=registry, images=identities, cell=outcome) + except BaseException as error: + result['error_type'] = type(error).__name__ + finally: + cleanup = recovery.cleanup(record) + result['cleanup'] = cleanup + result['cleanup_errors'] = owner.cleanup_errors + if not cleanup['pass'] or owner.cleanup_errors: + result.update(outcome='fail', checks_outcome='fail') + subject.CURRENT.verify() + data = subject.canonical(result) + clean, leaked = native.redact_credentials(data, owner.secret_values) + if leaked: + result = {'outcome': 'fail', 'checks_outcome': 'fail', 'error_type': 'CredentialOutput', + 'record_sha256': subject.RECORD_SHA} + clean = subject.canonical(result) + recovery.purge(args.work_dir) + summary_sha = subject.exclusive(pathlib.Path(args.work_dir) / 'summary.json', clean) + if cleanup['pass']: + watchdog.terminate() + watchdog.wait(timeout=5) + print(json.dumps({'summary_sha256': summary_sha, 'record_sha256': subject.RECORD_SHA, + 'manifest_sha256': subject.CURRENT.sha256, 'checks_outcome': result['checks_outcome'], + 'acceptance': 'requires independent captured-source audit after owner exit'}), flush=True) + require(result['checks_outcome'] == 'pass', 'checks failed; no acceptance') diff --git a/scripts/acceptance/condition-history/recover.py b/scripts/acceptance/condition-history/recover.py new file mode 100644 index 0000000..d5b0c1b --- /dev/null +++ b/scripts/acceptance/condition-history/recover.py @@ -0,0 +1,342 @@ +"""Captured, stdlib-only cleanup and independent final audit. No checkout import.""" +import argparse +import hashlib +import json +import os +import pathlib +import re +import signal +import stat +import subprocess +import sys +import time + +H = '4b2440a5618ccc1c98960ea2e1e62127042c3590' +PREFIX = 'sdk-rust-condition' +CELL = 'condition-history' +RUN = 'dev.durable-workflow.acceptance.run' +OWNER = 'dev.durable-workflow.acceptance.owner' +SUBJECT = 'dev.durable-workflow.acceptance.subject' +ROLE = 'dev.durable-workflow.acceptance.role' +RECORD_KEYS = {'schema', 'owner', 'token', 'cells', 'runner_commit', 'package_commit', 'manifest_sha256', + 'recovery_sha256', 'started_unix', 'active_unix', 'cleanup_unix', 'work_dir', + 'docker_endpoint', 'uid', 'gid', 'owner_process'} +ROLES = ('fetch', 'build', 'mysql', 'redis', 'bootstrap', 'runtime', 'queue', 'consumer') +PUBLIC_CONFIG = {'auths': {'https://index.docker.io/v1/': {}, 'ghcr.io': {}}} + + +def require(ok, label): + if not ok: + raise SystemExit(label) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def canonical(value): + return (json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n').encode() + + +def read(path, mode=0o600, limit=8 * 1024 * 1024): + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 + and stat.S_IMODE(before.st_mode) == mode and before.st_uid == os.getuid(), 'unsafe authority/artifact file') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as stream: + info = os.fstat(stream.fileno()) + require((info.st_dev, info.st_ino) == (before.st_dev, before.st_ino) and info.st_nlink == 1, 'authority file replaced') + data = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + require(len(data) <= limit and after.st_nlink == 1 + and (info.st_size, info.st_mtime_ns, info.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), 'authority file changed') + return data + + +def exclusive(path, data): + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'wb') as stream: + os.fchmod(stream.fileno(), 0o600) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + info = os.fstat(stream.fileno()) + require(info.st_nlink == 1 and stat.S_ISREG(info.st_mode) and info.st_size == len(data), 'unsafe final handoff') + require(read(path) == data, 'handoff changed') + return digest(data) + + +def process_identity(pid): + result = subprocess.run(['ps', '-p', str(pid), '-o', 'lstart='], capture_output=True, text=True, timeout=5) + if result.returncode or not result.stdout.strip(): + return None + return {'pid': pid, 'pgid': os.getpgid(pid), 'sid': os.getsid(pid), 'started': ' '.join(result.stdout.split())} + + +def group_members(pgid): + result = subprocess.run(['ps', '-axo', 'pid=,pgid='], capture_output=True, text=True, check=True, timeout=5) + return [int(fields[0]) for line in result.stdout.splitlines() + if len(fields := line.split()) == 2 and int(fields[1]) == pgid] + + +def make_record(args, manifest_sha, recovery_sha): + owner = process_identity(os.getpid()) + require(owner and owner['pid'] == owner['pgid'] == owner['sid'], 'isolated owner session required') + root = pathlib.Path(args.work_dir) + require(root.is_absolute() and root.resolve() == root, 'canonical work root required') + return {'schema': 'durable-workflow.condition-run/v2', 'owner': PREFIX, 'token': args.run_token, + 'cells': [CELL], 'runner_commit': args.runner_commit, 'package_commit': H, + 'manifest_sha256': manifest_sha, 'recovery_sha256': recovery_sha, + 'started_unix': args.started_unix, 'active_unix': args.active_unix, + 'cleanup_unix': args.cleanup_unix, 'work_dir': str(root), + 'docker_endpoint': args.docker_endpoint, 'uid': os.getuid(), 'gid': os.getgid(), + 'owner_process': owner} + + +def validate_record(root, expected_record_sha, runner, manifest_sha, recovery_sha, token): + root = pathlib.Path(root) + require(root.is_absolute() and root.resolve() == root and not root.is_symlink(), 'redirected work root') + info = root.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() + and stat.S_IMODE(info.st_mode) == 0o700, 'wrong work root authority') + raw = read(root / 'run.json') + require(digest(raw) == expected_record_sha, 'run record digest mismatch') + record = json.loads(raw) + require(set(record) == RECORD_KEYS and record['schema'] == 'durable-workflow.condition-run/v2', 'wrong run record schema') + require(record['owner'] == PREFIX and record['token'] == token and re.fullmatch('[0-9a-f]{16}', token) + and record['cells'] == [CELL], 'wrong recovery ownership') + require(record['runner_commit'] == runner and re.fullmatch('[0-9a-f]{40}', runner) + and record['package_commit'] == H and record['manifest_sha256'] == manifest_sha + and record['recovery_sha256'] == recovery_sha, 'wrong recovery source subject') + require(record['work_dir'] == str(root) and record['uid'] == os.getuid() and record['gid'] == os.getgid() + and record['docker_endpoint'].startswith('unix:///'), 'wrong recovery placement') + require(all(type(record[k]) is int for k in ('started_unix', 'active_unix', 'cleanup_unix')) + and record['cleanup_unix'] == record['started_unix'] + 1800 + and record['active_unix'] == record['cleanup_unix'] - 60, 'wrong recovery deadlines') + process = record['owner_process'] + require(set(process) == {'pid', 'pgid', 'sid', 'started'} and type(process['pid']) is int + and process['pid'] > 1 and process['pid'] == process['pgid'] == process['sid'] + and isinstance(process['started'], str) and process['started'], 'wrong captured owner process') + snapshot = root / 'snapshot' + require(not snapshot.is_symlink() and stat.S_ISDIR(snapshot.lstat().st_mode), 'unsafe recovery snapshot') + raw_manifest = read(snapshot / 'manifest.json', mode=0o400) + require(digest(raw_manifest) == manifest_sha, 'recovery manifest changed') + value = json.loads(raw_manifest) + require(set(value) == {'schema', 'runner_commit', 'package_commit', 'files'} + and value['schema'] == 'durable-workflow.condition-subject/v1' + and value['runner_commit'] == runner and value['package_commit'] == H, 'wrong sealed manifest authority') + expected = value['files'] + require(expected['scripts/acceptance/condition-history/recover.py']['sha256'] == recovery_sha, 'recovery source drift') + seen = set() + allowed_dirs = {p.as_posix() for name in expected for p in pathlib.PurePosixPath(name).parents if str(p) != '.'} + def unreadable(error): + raise SystemExit('unreadable recovery snapshot') + for parent, dirs, files in os.walk(snapshot, followlinks=False, onerror=unreadable): + for name in dirs: + path = pathlib.Path(parent) / name + require(not path.is_symlink() and stat.S_ISDIR(path.lstat().st_mode) + and path.relative_to(snapshot).as_posix() in allowed_dirs, 'redirected or extra snapshot directory') + for name in files: + path = pathlib.Path(parent) / name + relative = path.relative_to(snapshot).as_posix() + seen.add(relative) + if relative == 'manifest.json': + continue + require(relative in expected, 'unexpected recovery source member') + data = read(path, mode=0o400) + require(len(data) == expected[relative]['bytes'] and digest(data) == expected[relative]['sha256'], 'sealed recovery source changed') + require(seen == set(expected) | {'manifest.json'}, 'missing recovery source member') + return record + + +def docker(record, command, check=True): + root = pathlib.Path(record['work_dir']) + require(json.loads(read(root / 'recovery-docker/config.json')) == PUBLIC_CONFIG, 'anonymous Docker configuration changed') + env = {'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'HOME': str(root / 'public-home'), + 'DOCKER_CONFIG': str(root / 'recovery-docker'), 'DOCKER_HOST': record['docker_endpoint']} + remaining = record['cleanup_unix'] - time.time() + require(remaining > 0, 'cleanup deadline exhausted') + result = subprocess.run(['docker', *command], capture_output=True, text=True, timeout=min(5, remaining), env=env) + require(not check or result.returncode == 0, 'recovery Docker query/removal failed') + return result + + +def object_names(record): + base = PREFIX + '-' + record['token'] + task = base + '-' + CELL + # The only names this invocation can own, including partial build creates. + return task, {r: base + '-' + r if r in ('fetch', 'build') else task + '-' + r for r in ROLES} + + +def objects(record, kind): + command = {'container': ['ps', '-aq'], 'network': ['network', 'ls', '-q'], 'volume': ['volume', 'ls', '-q']}[kind] + return docker(record, command + ['--filter', 'label=' + OWNER + '=' + PREFIX, + '--filter', 'label=' + RUN + '=' + record['token']]).stdout.split() + + +def owned(record, kind, identifier): + labels = '.Config.Labels' if kind == 'container' else '.Labels' + template = '{"labels":{{json %s}},"name":{{json .Name}},"id":{{json .Id}}}' % labels + result = docker(record, [kind, 'inspect', '--format', template, identifier], check=False) + if result.returncode: + require(any(x in result.stderr.lower() for x in ('no such object', 'no such container', 'no such network')), 'uncertain object ownership') + return None + frame = json.loads(result.stdout) + labels, name = frame['labels'], frame['name'].removeprefix('/') + task, names_ = object_names(record) + role = labels.get(ROLE) + require(labels.get(OWNER) == PREFIX and labels.get(RUN) == record['token'] + and labels.get(SUBJECT) == record['manifest_sha256'], 'foreign object authority') + require((kind == 'network' and role == 'network' and name == task) + or (kind == 'container' and role in names_ and name == names_[role]), 'unknown owned object name') + require(isinstance(frame['id'], str) and re.fullmatch('[0-9a-f]{64}', frame['id']), 'invalid immutable object ID') + return frame['id'] + + +def cleanup(record): + errors = [] + for kind in ('container', 'network'): + try: + identifiers = objects(record, kind) + except BaseException: + errors.append('query-' + kind) + continue + for identifier in identifiers: + try: + object_id = owned(record, kind, identifier) + if object_id: + docker(record, ['rm', '-fv', object_id] if kind == 'container' else ['network', 'rm', object_id]) + except BaseException: + errors.append('remove-' + kind) + remaining = {} + for kind in ('container', 'network', 'volume'): + try: + remaining[kind] = objects(record, kind) + except BaseException: + remaining[kind] = None + errors.append('verify-' + kind) + return {'pass': not errors and all(v == [] for v in remaining.values()), 'errors': errors, 'remaining': remaining} + + +def purge(root): + import shutil + parent = pathlib.Path(root) / 'cells' + require(not parent.is_symlink(), 'redirected Cell parent') + target = parent / CELL + if target.is_symlink(): + target.unlink() + elif target.exists(): + shutil.rmtree(target) + require(not os.path.lexists(target), 'Cell purge survivor') + + +def stop_owner(record): + expected = record['owner_process'] + current = process_identity(expected['pid']) + if current is not None: + require(current == expected, 'owner PID epoch changed; refusing foreign process') + os.killpg(expected['pgid'], signal.SIGTERM) + until = time.monotonic() + 5 + while group_members(expected['pgid']) and time.monotonic() < until: + time.sleep(0.1) + if group_members(expected['pgid']): + os.killpg(expected['pgid'], signal.SIGKILL) + require(not group_members(expected['pgid']), 'owner group survived or cannot be attributed') + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=('recover', 'audit')) + for key in ('work-dir', 'record-sha256', 'manifest-sha256', 'runner-commit', 'package-commit', 'run-token'): + parser.add_argument('--' + key, required=True) + parser.add_argument('--summary-sha256') + parser.add_argument('--at', type=int) + args = parser.parse_args(argv) + require(args.package_commit == H, 'wrong recovery package source') + original = sys.orig_argv + require('-I' in original and '-S' in original and '-c' in original, 'captured recovery program required') + source_sha = digest(original[original.index('-c') + 1].encode()) + record = validate_record(args.work_dir, args.record_sha256, args.runner_commit, + args.manifest_sha256, source_sha, args.run_token) + config_dir = pathlib.Path(args.work_dir) / 'recovery-docker' + if not config_dir.exists(): + config_dir.mkdir(mode=0o700) + exclusive(config_dir / 'config.json', canonical(PUBLIC_CONFIG)) + require(not config_dir.is_symlink() and json.loads(read(config_dir / 'config.json')) == PUBLIC_CONFIG, + 'wrong captured recovery configuration') + if args.mode == 'recover': + if args.at is not None: + exclusive(pathlib.Path(args.work_dir) / 'recovery-ready.json', canonical({ + 'record_sha256': args.record_sha256, 'manifest_sha256': args.manifest_sha256, + 'recovery_sha256': source_sha, 'process': process_identity(os.getpid())})) + root_fd = os.open(args.work_dir, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + errors = [] + try: + if args.at is not None: + require(args.at == record['active_unix'], 'wrong recovery wake deadline') + time.sleep(max(0, args.at - time.time())) + # A later file drift fails evidence but cannot replace the captured + # authority/code or prevent its already-authorized exact cleanup. + try: + validate_record(args.work_dir, args.record_sha256, args.runner_commit, + args.manifest_sha256, source_sha, args.run_token) + except BaseException: + errors.append('source-or-record-drift') + try: + stop_owner(record) + except BaseException: + errors.append('owner-process-uncertain') + result = cleanup(record) + try: + import shutil + parent = os.open('cells', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root_fd) + try: + info = os.stat(CELL, dir_fd=parent, follow_symlinks=False) + if stat.S_ISLNK(info.st_mode):os.unlink(CELL, dir_fd=parent) + else:shutil.rmtree(CELL, dir_fd=parent) + except FileNotFoundError: + pass + finally: + os.close(parent) + except FileNotFoundError: + pass + except BaseException: + errors.append('cell-purge-uncertain') + result['errors'] += errors + result['pass'] = result['pass'] and not errors + exclusive(pathlib.Path(args.work_dir) / 'recovery-audit.json', canonical(result)) + require(result['pass'], 'recovery incomplete') + finally: + os.close(root_fd) + else: + require(args.summary_sha256 and re.fullmatch('[0-9a-f]{64}', args.summary_sha256), 'expected immutable summary digest required') + raw = read(pathlib.Path(args.work_dir) / 'summary.json') + require(digest(raw) == args.summary_sha256, 'summary handoff digest mismatch') + summary = json.loads(raw) + require(summary.get('record_sha256') == args.record_sha256 + and summary.get('runner_commit') == args.runner_commit and summary.get('package_commit') == H + and summary.get('manifest_sha256') == args.manifest_sha256, 'summary subject mismatch') + remaining = {kind: objects(record, kind) for kind in ('container', 'network', 'volume')} + require(all(v == [] for v in remaining.values()), 'independent object survivor') + require(not group_members(record['owner_process']['pgid']), 'independent owner process survivor') + watcher_raw = read(pathlib.Path(args.work_dir) / 'recovery-process.json') + require(digest(watcher_raw) == summary.get('recovery_process_sha256'), 'recovery-process handoff changed') + watcher = json.loads(watcher_raw) + require(set(watcher) == {'source_sha256', 'process'} and watcher['source_sha256'] == source_sha, + 'wrong captured recovery-process receipt') + process = watcher['process'] + require(set(process) == {'pid', 'pgid', 'sid', 'started'} and type(process['pid']) is int + and process['pid'] > 1 and process['pid'] == process['pgid'] == process['sid'] + and isinstance(process['started'], str) and process['started'], 'wrong recovery-process identity') + require(not group_members(watcher['process']['pgid']), 'independent recovery process survivor') + require(summary.get('checks_outcome') == 'pass', 'source checks did not pass') + result = {'outcome': 'pass', 'record_sha256': args.record_sha256, + 'manifest_sha256': args.manifest_sha256, 'summary_sha256': args.summary_sha256, + 'remaining': remaining, 'owner_process_absent': True, 'recovery_process_absent': True} + exclusive(pathlib.Path(args.work_dir) / 'independent-audit.json', canonical(result)) + print(json.dumps(result)) + + +if __name__ == '__main__': + main() diff --git a/scripts/acceptance/condition-history/resources.py b/scripts/acceptance/condition-history/resources.py new file mode 100644 index 0000000..4cec97c --- /dev/null +++ b/scripts/acceptance/condition-history/resources.py @@ -0,0 +1,216 @@ +"""Literal Docker role profiles and effective readbacks. Tests supply a fake CLI.""" +import hashlib +import json +import os +import pathlib +import re +import subprocess +import time + +RUN = 'dev.durable-workflow.acceptance.run' +OWNER = 'dev.durable-workflow.acceptance.owner' +SUBJECT = 'dev.durable-workflow.acceptance.subject' +ROLE = 'dev.durable-workflow.acceptance.role' +PREFIX = 'sdk-rust-condition' +CELL = 'condition-history' +PROFILES = { + 'fetch': (2_000_000_000, 3 * 1024**3, 1024, True, ()), + 'build': (2_000_000_000, 3 * 1024**3, 1024, True, ()), + 'mysql': (1_000_000_000, 1280 * 1024**2, 512, False, ('CHOWN', 'DAC_OVERRIDE', 'SETGID', 'SETUID')), + 'redis': (250_000_000, 128 * 1024**2, 256, False, ('CHOWN', 'SETGID', 'SETUID')), + 'bootstrap': (750_000_000, 768 * 1024**2, 256, False, ('DAC_OVERRIDE',)), + 'runtime': (750_000_000, 768 * 1024**2, 256, False, ('CHOWN', 'DAC_OVERRIDE', 'KILL', 'SETGID', 'SETUID')), + 'queue': (500_000_000, 512 * 1024**2, 256, False, ()), + 'consumer': (500_000_000, 768 * 1024**2, 256, True, ()), +} + + +def require(ok, label): + if not ok: + raise SystemExit(label) + + +def names(token): + base = PREFIX + '-' + token + task = base + '-' + CELL + return task, {r: base + '-' + r if r in ('fetch', 'build') else task + '-' + r for r in PROFILES} + + +def environment(args): + root = pathlib.Path(args.work_dir) + return {'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'HOME': str(root / 'public-home'), + 'DOCKER_HOST': args.docker_endpoint, 'DOCKER_CONFIG': str(root / 'public-docker')} + + +def call(args, line, timeout=120, check=True, **kwargs): + remaining = int(args.active_unix - time.time()) + require(remaining > 0, 'active deadline exhausted') + result = subprocess.run(line, capture_output=True, timeout=min(timeout, remaining), + env=environment(args), **dict({'text': True}, **kwargs)) + require(not check or result.returncode == 0, 'bounded command failed') + return result + + +def preflight_collisions(args): + """Every query is read-only and precedes work/snapshot/registry creation.""" + task, roles = names(args.run_token) + filters = ['--filter', 'label=' + RUN + '=' + args.run_token, '--filter', 'label=' + OWNER + '=' + PREFIX] + for command in (['docker', 'ps', '-aq'], ['docker', 'network', 'ls', '-q'], ['docker', 'volume', 'ls', '-q']): + require(not call(args, command + filters).stdout.split(), 'preexisting owner/run object') + for kind, candidates in [('container', roles.values()), ('network', [task])]: + for name in candidates: + result = call(args, ['docker', kind, 'inspect', '--format', '{{.Id}}', name], check=False) + if result.returncode == 0: + raise SystemExit('preexisting exact object name') + require(any(s in result.stderr.lower() for s in ('no such object', 'no such container', 'no such network')), + 'collision query is uncertain') + + +def caps(values): + return sorted(v.removeprefix('CAP_').upper() for v in values or []) + + +def env_map(values): + pairs = [v.split('=', 1) for v in values or []] + require(all(len(p) == 2 for p in pairs), 'malformed environment readback') + return dict(pairs) + + +class Docker: + def __init__(self, args, subject, record): + self.args, self.subject, self.record = args, subject, record + self.task, self.names = names(args.run_token) + self.images, self.receipts, self.created = {}, [], [] + + def call(self, line, **kwargs): + self.subject.verify() + return call(self.args, line, **kwargs) + + def labels(self, role): + return {RUN: self.args.run_token, OWNER: PREFIX, SUBJECT: self.subject.sha256, ROLE: role} + + def image(self, reference): + if reference not in self.images: + data = json.loads(self.call(['docker', 'image', 'inspect', reference]).stdout)[0] + require(data['Os'] == 'linux' and data['Architecture'] == 'amd64', 'wrong image platform') + content = json.dumps({'layers': data['RootFS']['Layers'], 'config': data['Config']}, sort_keys=True).encode() + self.images[reference] = {'raw': data, 'sha256': hashlib.sha256(content).hexdigest()} + return self.images[reference] + + def profile(self, role, image, command, user, credentials, fixture, state=None, env=None, alias=None, deadline=None): + require(role in PROFILES and credentials in ('none', 'auth', 'server', 'both'), 'unknown role/credential profile') + require(deadline is not None and deadline + 10 <= self.args.active_unix, 'role exceeds active deadline') + cpus, memory, pids, readonly, added = PROFILES[role] + metadata = self.image(image)['raw']['Config'] + child = (metadata.get('Entrypoint') or []) + (command or metadata.get('Cmd') or []) + require(child and all(isinstance(v, str) for v in child), 'invalid image entrypoint') + uid, gid = user.split(':') + require(uid.isdecimal() and gid.isdecimal() and (role not in ('fetch', 'build', 'consumer') or int(uid) != 0), 'unsafe role user') + tmpfs = {'/credentials': 'mode=0700,uid=%s,gid=%s,size=64k,noexec,nosuid,nodev' % (uid, gid), + '/tmp': 'mode=1777,size=64m,nosuid,nodev'} + binds = [] + if role in ('fetch', 'build'): + binds.append((str(fixture), '/work', role == 'fetch')) + if role == 'build': + binds += [(str(fixture / '.cargo'), '/work/.cargo', True), + (str(fixture / 'target'), '/work/target', True)] + elif role == 'consumer': + binds += [(str(fixture), '/work', False), (str(fixture / 'target/debug'), '/fixture', False), + (str(state), '/state', True)] + elif role == 'mysql': + tmpfs['/var/lib/mysql'] = 'mode=0750,size=512m,nosuid,nodev,noexec' + elif role == 'redis': + tmpfs['/data'] = 'mode=0750,size=64m,nosuid,nodev,noexec' + if role in ('bootstrap', 'runtime', 'queue'): + tmpfs['/app/bootstrap/cache'] = 'mode=0777,size=8m,noexec,nosuid,nodev' + declared = set((metadata.get('Volumes') or {}).keys()) + require(declared <= set(tmpfs) | {b[1] for b in binds}, 'uncovered image volume') + network = 'bridge' if role == 'fetch' else 'none' if role == 'build' else self.task + program = self.subject.read('scripts/acceptance/condition-history/role-supervisor.sh').decode() + argv = ['-c', program, '--', program, str(deadline), '10', credentials] + child + expected_env = env_map(metadata.get('Env')) + expected_env.update(env or {}) + return {'role': role, 'name': self.names[role], 'image': image, 'user': user, 'credentials': credentials, + 'cpus': cpus, 'memory': memory, 'pids': pids, 'readonly': readonly, 'cap_add': list(added), + 'tmpfs': tmpfs, 'binds': binds, 'network': network, 'alias': alias, 'deadline': deadline, + 'argv': argv, 'env': expected_env, 'explicit_env': env or {}} + + def create_arguments(self, spec): + line = ['docker', 'create', '--init', '--name', spec['name'], '--platform', 'linux/amd64', + '--user', spec['user'], '--cpus', str(spec['cpus'] / 1_000_000_000), + '--memory', str(spec['memory']), '--memory-swap', str(spec['memory']), '--pids-limit', str(spec['pids']), + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--network', spec['network'], + '--entrypoint', '/bin/sh'] + if spec['readonly']: + line += ['--read-only'] + for cap in spec['cap_add']: + line += ['--cap-add', cap] + for key, value in self.labels(spec['role']).items(): + line += ['--label', key + '=' + value] + for destination, options in spec['tmpfs'].items(): + line += ['--tmpfs', destination + ':' + options] + for source, destination, writable in spec['binds']: + line += ['--mount', 'type=bind,src=%s,dst=%s%s' % (source, destination, '' if writable else ',readonly')] + if spec['alias']: + line += ['--network-alias', spec['alias']] + if spec['role'] in ('fetch', 'build'): + line += ['-w', '/work'] + for key, value in spec['explicit_env'].items(): + line += ['-e', key + '=' + value] + return line + [spec['image']] + spec['argv'] + + def validate(self, spec, data): + host, config = data['HostConfig'], data['Config'] + require(data['Name'] == '/' + spec['name'] and re.fullmatch('[0-9a-f]{64}', data['Id']), 'wrong effective name/ID') + require(config['Image'] == spec['image'] and self.image(data['Image'])['sha256'] == self.image(spec['image'])['sha256'], 'wrong effective image') + require(data['State']['Running'] is True and type(data['State']['Pid']) is int and data['State']['Pid'] > 0, 'role not held for readback') + require(config.get('WorkingDir', '') == ('/work' if spec['role'] in ('fetch', 'build') else self.image(spec['image'])['raw']['Config'].get('WorkingDir', '')), 'wrong effective working directory') + require(config['User'] == spec['user'] and config['Entrypoint'] == ['/bin/sh'] + and config['Cmd'] == spec['argv'], 'wrong user or deadline supervisor') + require(host['NanoCpus'] == spec['cpus'] and host['Memory'] == spec['memory'] + and host['MemorySwap'] == spec['memory'] and host['PidsLimit'] == spec['pids'], 'wrong effective resource cap') + require(host['ReadonlyRootfs'] is spec['readonly'] and host['Init'] is True + and host['Privileged'] is False, 'wrong root/init/privilege mode') + require(caps(host['CapDrop']) == ['ALL'] and caps(host['CapAdd']) == caps(spec['cap_add']), 'wrong capability set') + require(sorted(v.removesuffix(':true') for v in host['SecurityOpt']) == ['no-new-privileges'], 'wrong security options') + require(host['NetworkMode'] == spec['network'] and not host.get('PortBindings') and not host.get('PublishAllPorts') + and not host.get('Devices') and not host.get('VolumesFrom') + and host.get('PidMode', '') != 'host' and host.get('IpcMode', 'private') != 'host', 'wrong effective network/device sharing') + if spec['network'] == self.task: + network = json.loads(self.call(['docker', 'network', 'inspect', self.task]).stdout)[0] + require(network['Internal'] is True and all(network['Labels'].get(k) == v for k, v in self.labels('network').items()), 'wrong internal network authority') + aliases = set(data['NetworkSettings']['Networks'].get(self.task, {}).get('Aliases') or []) + require(not spec['alias'] or spec['alias'] in aliases, 'missing effective network alias') + require(set(data['NetworkSettings']['Networks']) == {spec['network']}, 'unexpected attached network') + require(all(config['Labels'].get(k) == v for k, v in self.labels(spec['role']).items()), 'wrong effective ownership') + require(env_map(config['Env']) == spec['env'], 'wrong effective environment') + require(host.get('Tmpfs', {}) == spec['tmpfs'], 'wrong tmpfs destinations/options') + actual = [(m['Source'], m['Destination'], m['RW']) for m in data['Mounts'] if m['Type'] == 'bind'] + require(sorted(actual) == sorted(spec['binds']) and all(m['Type'] in ('bind', 'tmpfs') for m in data['Mounts']), 'wrong bind/volume mounts') + require({m['Destination'] for m in data['Mounts'] if m['Type'] == 'tmpfs'} == set(spec['tmpfs']) + and len(data['Mounts']) == len(spec['tmpfs']) + len(spec['binds']), 'missing, duplicate or unexpected mount') + return {'role': spec['role'], 'container_id': data['Id'], 'image': spec['image'], + 'image_content_sha256': self.image(spec['image'])['sha256'], 'user': spec['user'], + 'memory': spec['memory'], 'memory_swap': spec['memory'], 'nano_cpus': spec['cpus'], + 'pids': spec['pids'], 'readonly': spec['readonly'], 'cap_drop': ['ALL'], 'cap_add': spec['cap_add'], + 'security_opt': ['no-new-privileges'], 'network': spec['network'], 'deadline_unix': spec['deadline'], + 'supervisor_argv_sha256': hashlib.sha256(json.dumps(spec['argv']).encode()).hexdigest(), + 'mount_destinations': sorted(set(spec['tmpfs']) | {m[1] for m in spec['binds']}), + 'labels': self.labels(spec['role'])} + + def start(self, spec, token='', key=''): + self.subject.verify() + self.created.append(spec['name']) # before a possibly partial daemon create + self.call(self.create_arguments(spec)) + self.call(['docker', 'start', spec['name']]) + data = json.loads(self.call(['docker', 'inspect', spec['name']]).stdout)[0] + receipt = self.validate(spec, data) + self.receipts.append(receipt) + # No image command starts before the effective independent readback. + self.call(['docker', 'exec', '-i', spec['name'], '/bin/sh', '-c', + 'umask 077; IFS= read -r token && IFS= read -r key && ' + '{ [ -z "$token" ] || printf %s "$token" > /credentials/token; } && ' + '{ [ -z "$key" ] || printf %s "$key" > /credentials/server-key; } && ' + ': > /credentials/ready.tmp && mv /credentials/ready.tmp /credentials/ready'], + input=token + '\n' + key + '\n', timeout=20) + return spec['name'] diff --git a/scripts/acceptance/condition-history/role-supervisor.sh b/scripts/acceptance/condition-history/role-supervisor.sh new file mode 100644 index 0000000..f884117 --- /dev/null +++ b/scripts/acceptance/condition-history/role-supervisor.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -eu +source=$1 +shift +if [ "${1-}" = '--child' ]; then + shift + credentials=$1 + shift + unset DW_AUTH_TOKEN DW_SERVER_KEY APP_KEY + # All roles, including builds/databases, wait for verified effective caps. + while [ ! -f /credentials/ready ]; do sleep 0.1; done + case "$credentials" in + none) ;; + auth|server|both) ;; + *) exit 125;; + esac + if [ "$credentials" = auth ] || [ "$credentials" = both ]; then + DW_AUTH_TOKEN=$(cat /credentials/token) + case "$DW_AUTH_TOKEN" in fixture-*) hex=${DW_AUTH_TOKEN#fixture-};; *) exit 125;; esac + [ "${#hex}" -eq 48 ] || exit 125 + case "$hex" in *[!0-9a-f]*) exit 125;; esac + export DW_AUTH_TOKEN + fi + if [ "$credentials" = server ] || [ "$credentials" = both ]; then + DW_SERVER_KEY=$(cat /credentials/server-key) + case "$DW_SERVER_KEY" in base64:*) key=${DW_SERVER_KEY#base64:};; *) exit 125;; esac + [ "${#key}" -eq 44 ] || exit 125 + case "$key" in *[!a-zA-Z0-9+/=]*) exit 125;; esac + export DW_SERVER_KEY + fi + exec "$@" +fi +deadline=$1 +grace=$2 +shift 2 +case "$deadline:$grace" in *[!0-9:]*|:*|*:) exit 125;; esac +remaining=$((deadline - $(date +%s))) +[ "$remaining" -gt 0 ] || exit 124 +# Re-execute captured inline bytes, never a mutable bind-mounted script path. +exec timeout --signal=TERM --kill-after="${grace}s" "${remaining}s" \ + /bin/sh -c "$source" -- "$source" --child "$@" diff --git a/scripts/acceptance/condition-history/run.py b/scripts/acceptance/condition-history/run.py index 45a4a9f..9fc31a4 100644 --- a/scripts/acceptance/condition-history/run.py +++ b/scripts/acceptance/condition-history/run.py @@ -1,367 +1,33 @@ #!/usr/bin/env python3 -"""One explicitly admitted public-package condition-history acceptance. - -Source preparation and stubbed tests do not authorize invoking this runner. -See README.md for its fixed tuple, command and remaining review gate. -""" +"""Captured entry point. Invoke these Git-object bytes with Python -I -S -c.""" import argparse -import importlib.util -import json -import os import pathlib import re -import shutil import subprocess import sys -import time -import tomllib import types -import urllib.request - -HERE = pathlib.Path(__file__).resolve().parent -REPO = HERE.parents[2] -spec = importlib.util.spec_from_file_location('installed_native', HERE.parent / 'installed-native/run.py') -native = importlib.util.module_from_spec(spec) -spec.loader.exec_module(native) -spec = importlib.util.spec_from_file_location('condition_history_check', HERE / 'check_history.py') -comparator = importlib.util.module_from_spec(spec) -spec.loader.exec_module(comparator) - -H = '4b2440a5618ccc1c98960ea2e1e62127042c3590' -CRATE_SHA = '620a8287af73c363dac61f0a91f20971a39f3b8cdc9513fbea85cdbd95a74ba4' -RUST = 'rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081' -SERVER = 'ghcr.io/durable-workflow/server@sha256:980fe0a052089ced34ec3fcd2be2ea4e77436861d8fb537fb6654cb652a53e07' -SERVER_INDEX = 'sha256:3848d5d5cd1b23bd2881e3fdecea64cc44b72e6fb812e577c2db6d0872d065dd' -SERVER_SOURCE = 'fa3fc2f2f6b8935ea4a6eea47f6a9221e2adafff' -MYSQL = 'mysql@sha256:80f4933e3835f9dc4d35a28ec500d7986cb4414e6c6821c5461239cb7beb8995' -REDIS = 'redis@sha256:ca0acbb137c1dc3339c8b147a58fd6f42775d4599327b50e7b116c23de501af2' -INPUTS = { - 'tests/replay_regression_corpus.rs': 'b54be69c5b82ec6ad32bd2871c3451f182185fa01ca3316a1c3aee935c48c9cc', - 'tests/fixtures/replay-regressions/two-member-condition-group-server-reopen.json': - '815526c761ce314960335b0f7cafaa0aa60fa519ddd08252451a7a5ebdca821d', - 'tests/fixtures/replay-regressions/timer-separated-condition-group-server-reopen.json': - 'f86ecb0cc57634a3eb4c87ffbd4fa55555b82c2794a16cd7b4ce9d33e009d0f6', - 'tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json': - '50a630596b5d85a392a33cc595fe58a513f59f6ebdbba928fa22be4ea0fb9075', - 'tests/fixtures/replay-regressions/condition-orphaned-duplicate-timeout-keeps-wait.json': - '012790d1fc23fe11ef9208d3a8243dc282d160903f7df98396c77e1f8fd898f7', -} -CELL = 'condition-history' -PREFIX = 'sdk-rust-condition' -LIMITS = {'mysql': ('1', '1280m'), 'redis': ('0.25', '128m'), - 'bootstrap': ('0.75', '768m'), 'runtime': ('0.75', '768m'), - 'queue': ('0.5', '512m'), 'consumer': ('0.5', '768m')} - - -def require(condition, label): - if not condition: - raise SystemExit(label) - - -def git(*args): - return native.call(['git', '-C', str(REPO), *args]).stdout.strip() - - -def preflight(args): - # These are explicit checks: Python optimization cannot remove admission. - require(args.package_commit == H, 'wrong fixed package source') - require(re.fullmatch('[0-9a-f]{40}', args.runner_commit), 'invalid runner commit') - require(git('rev-parse', 'HEAD') == args.runner_commit, 'wrong runner head') - require(not git('status', '--porcelain', '--untracked-files=all'), 'runner checkout is not clean') - require(git('rev-parse', 'refs/tags/4.12.0^{commit}') == H, 'wrong immutable source tag') - require(not os.path.lexists(args.work_dir), 'work directory already exists') - require(re.fullmatch('[0-9a-f]{16}', args.run_token), 'invalid owner token') - require(args.cleanup_unix == args.started_unix + 1800 - and args.active_unix == args.cleanup_unix - 60, 'wrong absolute deadline envelope') - require(args.started_unix <= time.time() < args.started_unix + 30, 'stale launch timestamp') - # No command construction, pull, output directory or Popen precedes this. - - -def remaining(args): - seconds = int(args.active_unix - time.time()) - require(seconds > 10, 'active deadline exhausted') - return seconds - - -def command(args, argv, **kwargs): - return native.call(argv, timeout=min(kwargs.pop('timeout', 120), remaining(args)), **kwargs) - - -def export_consumer(fixture): - shutil.copytree(HERE / 'fixture', fixture) - for source, expected in INPUTS.items(): - raw = native.call(['git', '-C', str(REPO), 'show', H + ':' + source], text=False).stdout - require(native.hashlib.sha256(raw).hexdigest() == expected, 'fixed history input differs') - target = fixture / ('tests/official.rs' if source == 'tests/replay_regression_corpus.rs' else source) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - - -def build(args, fixture, owner, script, phase, network=None): - name = owner.base + '-' + phase - limit = remaining(args) - 10 - line = ['docker', 'run', '--rm', '--init', '--name', name, *owner.labels(name), - '--platform', 'linux/amd64', '--cpus', '2', '--memory', '3g', '--pids-limit', '1024', - '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', - '--user', '%d:%d' % (os.getuid(), os.getgid()), - '--mount', 'type=bind,src=%s,dst=/work' % fixture, '-w', '/work', - '-e', 'CARGO_HOME=/work/.cargo', '-e', 'CARGO_BUILD_JOBS=2', - '-e', 'CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback'] - if network: - line += ['--network', network] - try: - return command(args, line + [RUST, 'timeout', '--kill-after=10s', str(limit) + 's', 'sh', '-ec', script], - timeout=limit + 10) - finally: - owner.best_effort('build-remove', phase, native.remove_owned_container, name, owner) - - -def verify_crate(fixture): - lock = tomllib.loads((fixture / 'Cargo.lock').read_text()) - packages = [p for p in lock['package'] if p['name'] == 'durable-workflow'] - require(len(packages) == 1 and packages[0]['version'] == '4.12.0' - and packages[0]['source'] == native.CRATES_IO_SOURCE - and packages[0]['checksum'] == CRATE_SHA, 'wrong public package lock') - archives = list((fixture / '.cargo/registry/cache').glob('*/durable-workflow-4.12.0.crate')) - require(len(archives) == 1 and native.sha256(archives[0]) == CRATE_SHA, 'wrong downloaded archive') - with native.tarfile.open(archives[0]) as archive: - vcs = json.load(archive.extractfile('durable-workflow-4.12.0/.cargo_vcs_info.json')) - require(vcs['git']['sha1'] == H and not vcs['git'].get('dirty'), 'wrong public package VCS') - require(not (fixture / '.cargo/config.toml').exists(), 'registry replacement is forbidden') - return {'version': '4.12.0', 'source': H, 'archive_sha256': CRATE_SHA, - 'lock_sha256': native.sha256(fixture / 'Cargo.lock')} - - -def registry_identity(): - request = urllib.request.Request('https://index.crates.io/du/ra/durable-workflow', - headers={'User-Agent': 'durable-workflow-condition-history-acceptance'}) - with urllib.request.urlopen(request, timeout=30) as response: - versions = [json.loads(line) for line in response.read(8 * 1024 * 1024).splitlines()] - exact = [version for version in versions if version.get('vers') == '4.12.0'] - require(len(exact) == 1 and exact[0].get('cksum') == CRATE_SHA - and exact[0].get('yanked') is False, 'public registry identity refused') - return {'version': '4.12.0', 'sha256': CRATE_SHA, 'yanked': False} - - -class ConditionCell(native.Cell): - def __init__(self, *args): - super().__init__(*args) - self.receipts = [] - - def launch(self, line, name, image, child, user, credentials): - role = name.removeprefix(self.task + '-') - require(role in LIMITS, 'unknown condition role') - require(len(self.containers) < 5, 'condition role ceiling reached') - cpus, memory = LIMITS[role] - line[line.index('--cpus') + 1] = cpus - line[line.index('--memory') + 1] = memory - line += ['--platform', 'linux/amd64'] - if role in ('bootstrap', 'runtime', 'queue'): - line += ['--mount', 'type=bind,src=%s,dst=/state' % self.state] - name = super().launch(line, name, image, child, user, credentials) - # Targeted readback deliberately excludes environment and host paths. - template = ('{"id":{{json .Id}},"image":{{json .Image}},"memory":{{.HostConfig.Memory}},' - '"nano_cpus":{{.HostConfig.NanoCpus}},"pid":{{.State.Pid}},' - '"run":{{json (index .Config.Labels "' + native.RUN_LABEL + '")}},' - '"owner":{{json (index .Config.Labels "' + native.OWNER_LABEL + '")}},' - '"mounts":"{{range .Mounts}}{{.Type}}:{{.Destination}};{{end}}"}') - receipt = json.loads(native.call(['docker', 'inspect', '--format', template, name]).stdout) - require(receipt['run'] == self.harness_run.token and receipt['owner'] == self.harness_run.prefix, - 'role readback ownership mismatch') - require(receipt['memory'] == int(memory.removesuffix('m')) * 1024 * 1024 - and receipt['nano_cpus'] == int(float(cpus) * 1_000_000_000), 'role resource mismatch') - require('volume:' not in receipt['mounts'], 'anonymous volumes are forbidden') - receipt.update(role=role, deadline_unix=self.deadline, credentials=credentials) - self.receipts.append(receipt) - return name - - def start_php(self, token): - env = {'APP_ENV': 'testing', 'APP_DEBUG': 'false', 'DB_CONNECTION': 'mysql', - 'DB_HOST': 'mysql', 'DB_DATABASE': 'durable_workflow', 'DB_USERNAME': 'workflow', - 'DB_PASSWORD': 'workflow', 'REDIS_HOST': 'redis', 'QUEUE_CONNECTION': 'redis', - 'CACHE_STORE': 'redis', 'DW_AUTH_DRIVER': 'token', 'DW_WORKER_PROTOCOL_VERSION': '1.20'} - mysql = self.plain('mysql', MYSQL, [], alias='mysql', tmpfs='/var/lib/mysql', pids=512, - caps=('CHOWN', 'DAC_OVERRIDE', 'SETGID', 'SETUID'), - env={'MYSQL_DATABASE': 'durable_workflow', 'MYSQL_USER': 'workflow', - 'MYSQL_PASSWORD': 'workflow', 'MYSQL_ROOT_PASSWORD': 'root'}) - self.plain('redis', REDIS, [], alias='redis', tmpfs='/data', caps=('CHOWN', 'SETGID', 'SETUID')) - until = min(time.time() + 120, self.deadline - 15) - while native.call(['docker', 'exec', mysql, 'mysqladmin', 'ping', '-h', 'localhost', '-uworkflow', '-pworkflow'], - check=False).returncode: - require(time.time() < until, 'mysql readiness deadline') - time.sleep(1) - bootstrap = self.plain('bootstrap', SERVER, ['server-bootstrap'], env=env, - caps=('DAC_OVERRIDE',), credentials='server') - require(native.call(['docker', 'wait', bootstrap], timeout=max(1, self.deadline - time.time())).stdout.strip() == '0', - 'PHP bootstrap failed') - native.remove_owned_container(bootstrap, self.harness_run) - self.containers.remove(bootstrap) - self.plain('runtime', SERVER, [], env=env, alias='qualification-origin', credentials='both', - caps=('CHOWN', 'DAC_OVERRIDE', 'KILL', 'SETGID', 'SETUID')) - self.plain('queue', SERVER, ['php', 'artisan', 'queue:work', '--sleep=1', '--tries=3', '--max-time=600'], - env=env, user='1000:1000', credentials='server') - - def execute(self, args): - outcome = {'outcome': 'fail', 'deadline_unix': self.deadline} - try: - self.cell_path('state').mkdir(parents=True) - self.dir.chmod(0o777) - self.state.chmod(0o777) - command(args, ['docker', 'network', 'create', '--internal', *self.harness_run.labels(self.task), self.task]) - # No service roles exist during loopback replay; then exactly four - # service roles plus one combined driver/Worker. No proxy/repair. - replay = self.start('consumer', RUST, ['/fixture/condition-history', '--exact', - 'installed_conditions::installed_history_replay', '--nocapture']) - require(command(args, ['docker', 'wait', replay], timeout=300).stdout.strip() == '0', 'installed replay failed') - native.remove_owned_container(replay, self.harness_run) - self.containers.remove(replay) - self.start_php(self.token) - consumer = self.start('consumer', RUST, ['/fixture/condition-history', '--exact', - 'installed_conditions::installed_live_group', '--ignored', '--nocapture'], credentials='auth') - require(command(args, ['docker', 'wait', consumer], timeout=180).stdout.strip() == '0', 'live group failed') - finally: - stopped = self.remove_docker_objects() - try: - require(stopped, 'writer cleanup uncertain') - self.fence_artifacts() - require(not self.harness_run.cleanup_errors, 'evidence fence failed') - replay = json.loads(self.artifact_bytes(self.state / 'replay.json')) - live = json.loads(self.artifact_bytes(self.state / 'live.json')) - require(replay.get('outcome') == 'pass' and len(replay.get('cases', {})) == 9, 'replay report incomplete') - outcome.update(outcome='pass', replay=replay, live=live, history_check=comparator.check(live), - role_receipts=self.receipts) - except BaseException: - outcome['outcome'] = 'fail' - self.harness_run.best_effort('evidence-purge', CELL, self.purge_evidence) - raise - return outcome - - -def recover(root, token): - require(isinstance(token, str) and re.fullmatch('[0-9a-f]{16}', token), 'invalid recovery token') - record = json.loads(native.confined_child(root, 'run.json').read_text()) - require(record.get('owner') == PREFIX and record.get('token') == token - and record.get('cells') == [CELL], 'wrong recovery ownership') - owner = native.Run(PREFIX, token=token) - result = native.recover_run(root, owner, [CELL]) - require(result['pass'], 'recovery has survivors or uncertainty') - return result - - -def execute(args): - preflight(args) - args.work_dir.mkdir(mode=0o700) - owner = native.Run(PREFIX, token=args.run_token) - record = {'owner': PREFIX, 'token': owner.token, 'cells': [CELL], 'package_commit': H, - 'runner_commit': args.runner_commit, 'started_unix': args.started_unix, - 'active_unix': args.active_unix, 'cleanup_unix': args.cleanup_unix} - native.write_owner_only(args.work_dir / 'run.json', json.dumps(record)) - endpoint = native.call(['docker', 'context', 'inspect', '--format', '{{.Endpoints.docker.Host}}']).stdout.strip() - require(endpoint.startswith('unix://'), 'only the admitted local Docker daemon is supported') - public_config = args.work_dir / 'public-docker' - public_config.mkdir(mode=0o700) - native.write_owner_only(public_config / 'config.json', '{"auths":{}}\n') - # Keep the local daemon endpoint while removing registry logins and helpers. - # The endpoint is private evidence and is never serialized into the summary. - os.environ['DOCKER_HOST'] = endpoint - os.environ['DOCKER_CONFIG'] = str(public_config) - # The detached recovery process survives this client's death. Every Docker - # role also enforces its own absolute TERM/KILL deadline independently. - watchdog = subprocess.Popen([sys.executable, str(HERE / 'run.py'), 'recover', '--work-dir', str(args.work_dir), - '--run-token', owner.token, '--at', str(args.cleanup_unix)], start_new_session=True, - stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - result = {'outcome': 'fail', 'run': record} - try: - registry = registry_identity() - images = [RUST, SERVER, MYSQL, REDIS] - for image in images: - command(args, ['docker', 'pull', '--platform', 'linux/amd64', image], timeout=600) - identities = {image: native.image_identity(image) for image in images} - require(all(i['os'] == 'linux' and i['architecture'] == 'amd64' for i in identities.values()), 'wrong image architecture') - for image in images: - declared = json.loads(command(args, ['docker', 'image', 'inspect', '--format', '{{json .Config.Volumes}}', image]).stdout) or {} - covered = {MYSQL: {'/var/lib/mysql'}, REDIS: {'/data'}}.get(image, set()) - require(set(declared).issubset(covered), 'uncovered image volume declaration') - labels = json.loads(command(args, ['docker', 'image', 'inspect', '--format', '{{json .Config.Labels}}', SERVER]).stdout) - require(labels.get('org.opencontainers.image.revision') == SERVER_SOURCE, 'wrong PHP Server source') - fixture = args.work_dir / 'fixture' - export_consumer(fixture) - sources = {p.relative_to(fixture).as_posix(): native.sha256(p) - for p in fixture.rglob('*') if p.is_file()} - fetch = build(args, fixture, owner, 'cargo generate-lockfile && cargo fetch --locked', 'fetch') - (fixture / 'fetch.stdout').write_text(fetch.stdout) - package = verify_crate(fixture) - compiled = build(args, fixture, owner, 'rustc -V && cargo -V && cargo test --locked --offline --test conditions --no-run --message-format=json', - 'build', network='none') - (fixture / 'build.stdout').write_text(compiled.stdout) - (fixture / 'build.stderr').write_text(compiled.stderr) - toolchain = compiled.stdout.splitlines()[:2] - require(len(toolchain) == 2 and toolchain[0].startswith('rustc 1.86.0 ') - and toolchain[1].startswith('cargo 1.86.0 '), 'wrong actual Rust/Cargo toolchain') - artifacts = [json.loads(line) for line in compiled.stdout.splitlines() if line.startswith('{')] - executables = [p['executable'] for p in artifacts if p.get('reason') == 'compiler-artifact' - and p.get('target', {}).get('name') == 'conditions' and p.get('executable')] - require(len(executables) == 1 and executables[0].startswith('/work/target/debug/deps/conditions-'), 'wrong consumer executable') - binary = native.confined_child(fixture, executables[0].removeprefix('/work/')) - output = fixture / 'target/debug/condition-history' - shutil.copyfile(binary, output) - output.chmod(0o755) - binary_sha = native.sha256(output) - require(not native.run_objects('container', owner), 'build/runtime overlap') - cell = ConditionCell(types.SimpleNamespace(rust_image=RUST), CELL, args.work_dir, - {'executed_manifest': SERVER}, fixture, owner) - cell.deadline = min(int(time.time()) + 600, args.active_unix - 10) - outcome = cell.execute(args) - require(native.sha256(output) == binary_sha, 'consumer binary changed') - require(verify_crate(fixture) == package, 'public package changed') - require(all(native.sha256(fixture / path) == digest for path, digest in sources.items()), 'consumer source changed') - result.update(outcome='pass', package=package, binary_sha256=binary_sha, - input_sha256=INPUTS, consumer_source_sha256=sources, toolchain=toolchain, - registry=registry, images=identities, server_index=SERVER_INDEX, cell=outcome) - except BaseException as error: - result['error_type'] = type(error).__name__ - finally: - # Preserve only fenced Cell evidence during normal completion. Recovery - # purges interrupted cells, where the client no longer has secret bytes. - for kind in ('container', 'network'): - result['leftover_' + kind] = owner.best_effort('final-query', kind, native.run_objects, kind, owner) - result['cleanup_errors'] = owner.cleanup_errors - if owner.cleanup_errors or result['leftover_container'] != [] or result['leftover_network'] != []: - result['outcome'] = 'fail' - owner.best_effort('uncertain-recovery', CELL, recover, args.work_dir, owner.token) - data = json.dumps(result, indent=2).encode() - clean, leaked = native.redact_credentials(data, owner.secret_values) - if leaked: - result = {'outcome': 'fail', 'error_type': 'CredentialOutput'} - clean = json.dumps(result).encode() - owner.best_effort('credential-purge', CELL, recover, args.work_dir, owner.token) - (args.work_dir / 'summary.json').write_bytes(clean + b'\n') - # Cancel the backstop only after a direct zero-survivor readback. A - # failed query leaves it alive to perform the independent final sweep. - if native.run_objects('container', owner) == [] and native.run_objects('network', owner) == []: - watchdog.terminate() - watchdog.wait(timeout=10) - require(result['outcome'] == 'pass', 'acceptance failed; inspect bounded summary') - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('mode', choices=('run', 'recover')) - parser.add_argument('--work-dir', required=True, type=native.cli_path) - parser.add_argument('--run-token', required=True) - parser.add_argument('--package-commit') - parser.add_argument('--runner-commit') - parser.add_argument('--started-unix', type=int) - parser.add_argument('--active-unix', type=int) - parser.add_argument('--cleanup-unix', type=int) - parser.add_argument('--at', type=int) - args = parser.parse_args(argv) - if args.mode == 'recover': - if args.at is not None: - time.sleep(max(0, args.at - time.time())) - recover(args.work_dir, args.run_token) - else: - execute(args) +SUBJECT = 'scripts/acceptance/condition-history/subject.py' + + +def main(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('--checkout', required=True) + parser.add_argument('--runner-commit', required=True) + parser.add_argument('--package-commit', required=True) + args, _ = parser.parse_known_args() + if args.package_commit != '4b2440a5618ccc1c98960ea2e1e62127042c3590': + raise SystemExit('wrong fixed package source') + if not re.fullmatch('[0-9a-f]{40}', args.runner_commit): + raise SystemExit('invalid runner commit') + checkout = pathlib.Path(args.checkout).resolve(strict=True) + result = subprocess.run(['git', '--no-replace-objects', '-C', str(checkout), 'show', + args.runner_commit + ':' + SUBJECT], check=True, capture_output=True, timeout=20) + module = types.ModuleType('sealed_subject') + module.__file__ = 'git:' + args.runner_commit + ':' + SUBJECT + sys.modules['sealed_subject'] = module + exec(compile(result.stdout, module.__file__, 'exec'), module.__dict__) + module.bootstrap() if __name__ == '__main__': diff --git a/scripts/acceptance/condition-history/subject.py b/scripts/acceptance/condition-history/subject.py new file mode 100644 index 0000000..2190774 --- /dev/null +++ b/scripts/acceptance/condition-history/subject.py @@ -0,0 +1,237 @@ +"""Capture one closed Git-object subject; never import executable checkout files.""" +import argparse +import hashlib +import json +import os +import pathlib +import re +import stat +import subprocess +import sys +import time +import types + +PREFIX = 'scripts/acceptance/condition-history/' +H = '4b2440a5618ccc1c98960ea2e1e62127042c3590' +RUNTIME_FILES = tuple(PREFIX + p for p in ( + 'run.py', 'subject.py', 'engine.py', 'recover.py', 'resources.py', + 'role-supervisor.sh', 'check_history.py', 'fixture/Cargo.toml', 'fixture/tests/conditions.rs')) + ( + 'scripts/acceptance/installed-native/run.py', + 'scripts/acceptance/installed-native/cell-supervisor.sh') +PACKET_FILES = {p for p in RUNTIME_FILES if p.startswith(PREFIX)} | { + PREFIX + p for p in ('README.md', 'test_packet.py', 'test_subject.py', 'test_resources.py', 'test_audit.py')} +MODULES = {'engine': PREFIX + 'engine.py', 'resources': PREFIX + 'resources.py', + 'native': 'scripts/acceptance/installed-native/run.py', 'comparator': PREFIX + 'check_history.py', + 'recovery': PREFIX + 'recover.py'} + + +def require(ok, label): + if not ok: + raise SystemExit(label) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def canonical(value): + return (json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n').encode() + + +def git(checkout, *args, binary=False): + result = subprocess.run(['git', '--no-replace-objects', '--literal-pathspecs', '-C', str(checkout), *args], + check=True, capture_output=True, timeout=20) + return result.stdout if binary else result.stdout.decode().strip() + + +def manifest(checkout, revision): + require(re.fullmatch('[0-9a-f]{40}', revision), 'invalid runner commit') + listing = git(checkout, 'ls-tree', '-r', '-z', revision, '--', PREFIX, binary=True) + found = {entry.split(b'\t', 1)[1].decode() for entry in listing.split(b'\0') if entry} + require(found == PACKET_FILES, 'extra or missing packet Git member') + files, captured = {}, {} + for path in RUNTIME_FILES: + entry = git(checkout, 'ls-tree', revision, '--', path).split() + require(len(entry) == 4 and entry[0] in ('100644', '100755') and entry[1] == 'blob' + and entry[3] == path, 'nonregular Git member') + data = git(checkout, 'cat-file', 'blob', revision + ':' + path, binary=True) + require(len(data) <= 2 * 1024 * 1024, 'oversize sealed member') + captured[path] = data + files[path] = {'git_blob': entry[2], 'sha256': digest(data), 'bytes': len(data), 'mode': entry[0]} + value = {'schema': 'durable-workflow.condition-subject/v1', 'runner_commit': revision, + 'package_commit': H, 'files': files} + return value, captured + + +def regular(path, limit=2 * 1024 * 1024, mode=None): + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_uid == os.getuid(), 'nonregular, linked or foreign source/record') + if mode is not None: + require(stat.S_IMODE(before.st_mode) == mode, 'wrong private file mode') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as stream: + opened = os.fstat(stream.fileno()) + require((opened.st_dev, opened.st_ino) == (before.st_dev, before.st_ino) + and stat.S_ISREG(opened.st_mode) and opened.st_nlink == 1, 'source/record changed on open') + data = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + require(len(data) <= limit and (opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns) + and after.st_nlink == 1, 'source/record changed on read') + return data + + +def exclusive(path, data, mode=0o600): + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, mode) + with os.fdopen(fd, 'wb') as stream: + os.fchmod(stream.fileno(), mode) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size == len(data), 'unsafe exclusive handoff') + require(regular(path, max(len(data), 1), mode) == data, 'exclusive handoff changed') + return digest(data) + + +class Subject: + def __init__(self, value, captured, root): + self.value, self.captured, self.root = value, dict(captured), pathlib.Path(root) + self.sha256 = digest(canonical(value)) + self.loaded = {} + + def materialize(self): + self.root.mkdir(mode=0o700) + for path, data in self.captured.items(): + target = self.root / path + target.parent.mkdir(parents=True, exist_ok=True) + exclusive(target, data, 0o400) + exclusive(self.root / 'manifest.json', canonical(self.value), 0o400) + for parent, directories, _ in os.walk(self.root, topdown=False): + for directory in directories: + (pathlib.Path(parent) / directory).chmod(0o500) + self.root.chmod(0o500) + self.verify() + + def verify(self): + require(not self.root.is_symlink() and stat.S_ISDIR(self.root.lstat().st_mode), 'unsafe snapshot root') + found = set() + allowed_dirs = {p.as_posix() for name in self.captured for p in pathlib.PurePosixPath(name).parents if str(p) != '.'} + def unreadable(error): + raise SystemExit('unreadable snapshot directory') + for parent, directories, files in os.walk(self.root, followlinks=False, onerror=unreadable): + for name in directories: + path = pathlib.Path(parent) / name + require(not path.is_symlink() and stat.S_ISDIR(path.lstat().st_mode) + and path.relative_to(self.root).as_posix() in allowed_dirs, 'unsafe or extra snapshot directory') + for name in files: + path = pathlib.Path(parent) / name + relative = path.relative_to(self.root).as_posix() + found.add(relative) + data = regular(path, mode=0o400) + expected = canonical(self.value) if relative == 'manifest.json' else self.captured.get(relative) + require(expected is not None and data == expected, 'snapshot member changed or unexpected') + require(found == set(self.captured) | {'manifest.json'}, 'missing snapshot member') + + def read(self, path): + self.verify() + require(path in self.captured, 'member outside sealed subject') + return self.captured[path] + + def module(self, name): + require(name in MODULES, 'module outside sealed import set') + self.verify() + if name not in self.loaded: + path = MODULES[name] + module = types.ModuleType('sealed_' + name) + module.__file__ = str(self.root / path) + sys.modules[module.__name__] = module + exec(compile(self.captured[path], module.__file__, 'exec'), module.__dict__) + self.loaded[name] = module + return self.loaded[name] + + def export(self, prefix, destination): + self.verify() + for name, data in self.captured.items(): + if name.startswith(prefix): + target = destination / name.removeprefix(prefix) + target.parent.mkdir(parents=True, exist_ok=True) + exclusive(target, data) + + +def arguments(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=('run', 'recover', 'audit', 'manifest')) + for key in ('checkout', 'runner-commit', 'package-commit'): + parser.add_argument('--' + key, required=True) + parser.add_argument('--manifest-sha256') + parser.add_argument('--work-dir') + parser.add_argument('--run-token') + parser.add_argument('--docker-endpoint') + for key in ('started-unix', 'active-unix', 'cleanup-unix'): + parser.add_argument('--' + key, type=int) + parser.add_argument('--record-sha256') + parser.add_argument('--summary-sha256') + return parser.parse_args(argv) + + +def validate(args): + require(args.package_commit == H, 'wrong fixed package source') + require(git(args.checkout, 'rev-parse', 'HEAD') == args.runner_commit, 'wrong runner head') + require(not git(args.checkout, 'status', '--porcelain', '--untracked-files=all'), 'runner checkout is not clean') + require(git(args.checkout, 'rev-parse', 'refs/tags/4.12.0^{commit}') == H, 'wrong immutable package tag') + if args.mode != 'run': + return + require(args.work_dir and not os.path.lexists(args.work_dir), 'work directory is not fresh') + require(args.run_token and re.fullmatch('[0-9a-f]{16}', args.run_token), 'invalid run token') + require(args.docker_endpoint and args.docker_endpoint.startswith('unix:///'), 'local daemon endpoint required') + require(type(args.started_unix) is int and type(args.active_unix) is int and type(args.cleanup_unix) is int + and args.cleanup_unix == args.started_unix + 1800 + and args.active_unix == args.cleanup_unix - 60, 'wrong absolute deadline envelope') + require(args.started_unix <= time.time() < args.started_unix + 30, 'stale launch timestamp') + + +def bootstrap(): + global CURRENT, ARGS, RECORD, RECORD_SHA + args = arguments() + validate(args) + value, captured = manifest(args.checkout, args.runner_commit) + actual = digest(canonical(value)) + if args.mode == 'manifest': + print(json.dumps({'manifest_sha256': actual, 'manifest': value}, sort_keys=True)) + return + require(actual == args.manifest_sha256, 'wrong reviewed manifest') + original = sys.orig_argv + require('-I' in original and '-S' in original and '-c' in original, 'captured isolated entry point required') + entry = original[original.index('-c') + 1].encode() + require(digest(entry) == value['files'][PREFIX + 'run.py']['sha256'], 'mutable or changed entry point') + sys.dont_write_bytecode = True + if args.mode in ('recover', 'audit'): + program = captured[PREFIX + 'recover.py'].decode() + argv = [sys.executable, '-I', '-S', '-c', program, args.mode, '--work-dir', args.work_dir, + '--record-sha256', args.record_sha256 or '', '--manifest-sha256', actual, + '--runner-commit', args.runner_commit, '--package-commit', H, + '--run-token', args.run_token or ''] + if args.summary_sha256: + argv += ['--summary-sha256', args.summary_sha256] + os.execv(sys.executable, argv) + # Collision reads precede directory creation, registry work and every role. + temporary = Subject(value, captured, pathlib.Path(args.work_dir) / 'snapshot') + ARGS, CURRENT = args, temporary + module = types.ModuleType('sealed_resource_preflight') + module.__file__ = 'git:' + args.runner_commit + ':' + PREFIX + 'resources.py' + exec(compile(captured[PREFIX + 'resources.py'], module.__file__, 'exec'), module.__dict__) + module.preflight_collisions(args) + recovery = types.ModuleType('captured_recovery_preflight') + recovery.__file__ = 'git:' + args.runner_commit + ':' + PREFIX + 'recover.py' + exec(compile(captured[PREFIX + 'recover.py'], recovery.__file__, 'exec'), recovery.__dict__) + RECORD = recovery.make_record(args, actual, value['files'][PREFIX + 'recover.py']['sha256']) + RECORD_SHA = digest(canonical(RECORD)) + print(json.dumps({'private_launch': RECORD, 'record_sha256': RECORD_SHA}), flush=True) + pathlib.Path(args.work_dir).mkdir(mode=0o700) + exclusive(pathlib.Path(args.work_dir) / 'run.json', canonical(RECORD)) + for directory in ('public-home', 'public-docker'): + (pathlib.Path(args.work_dir) / directory).mkdir(mode=0o700) + exclusive(pathlib.Path(args.work_dir) / 'public-docker/config.json', canonical(recovery.PUBLIC_CONFIG)) + temporary.materialize() + CURRENT.module('engine').execute(args) diff --git a/scripts/acceptance/condition-history/test_audit.py b/scripts/acceptance/condition-history/test_audit.py new file mode 100644 index 0000000..7f5ba72 --- /dev/null +++ b/scripts/acceptance/condition-history/test_audit.py @@ -0,0 +1,105 @@ +"""Final handoff/recovery controls with synthetic records and a fake daemon.""" +import contextlib +import copy +import io +import json +import os +import pathlib +import tempfile +import unittest +from unittest import mock +import test_subject as controls +from test_subject import fixture_subject, recovery, remove_private, subject + + +class Audit(unittest.TestCase): + record = controls.SubjectControls.record + validate = controls.SubjectControls.validate + + def setUp(self): + self.root = pathlib.Path(tempfile.mkdtemp()).resolve() + self.addCleanup(remove_private, self.root) + self.sealed = fixture_subject(self.root) + self.authority = self.record() + self.validate(self.authority) + self.record_sha = subject.digest(subject.canonical(self.authority)) + self.program = self.sealed.captured[subject.PREFIX + 'recover.py'].decode() + self.watcher = {'source_sha256': self.authority['recovery_sha256'], + 'process': {'pid': 23456, 'pgid': 23456, 'sid': 23456, 'started': 'synthetic watcher'}} + watcher_sha = subject.exclusive(self.root / 'recovery-process.json', subject.canonical(self.watcher)) + self.summary = {'checks_outcome': 'pass', 'outcome': 'pending_independent_audit', + 'record_sha256': self.record_sha, 'runner_commit': self.authority['runner_commit'], + 'package_commit': subject.H, 'manifest_sha256': self.sealed.sha256, + 'recovery_process_sha256': watcher_sha} + self.summary_sha = subject.exclusive(self.root / 'summary.json', subject.canonical(self.summary)) + + def args(self, mode='audit'): + result = [mode, '--work-dir', str(self.root), '--record-sha256', self.record_sha, + '--manifest-sha256', self.sealed.sha256, '--runner-commit', self.authority['runner_commit'], + '--package-commit', subject.H, '--run-token', self.authority['token']] + if mode == 'audit':result += ['--summary-sha256', self.summary_sha] + return result + + def invoke(self, mode='audit', objects=None, groups=None): + with mock.patch.object(recovery.sys, 'orig_argv', ['python', '-I', '-S', '-c', self.program]), \ + mock.patch.object(recovery, 'objects', side_effect=objects or (lambda *a: [])), \ + mock.patch.object(recovery, 'group_members', side_effect=groups or (lambda *a: [])), \ + mock.patch.object(recovery, 'docker', side_effect=AssertionError('no actual daemon')), \ + contextlib.redirect_stdout(io.StringIO()): + recovery.main(self.args(mode)) + + def test_independent_audit_binds_exact_summary_and_both_processes(self): + self.invoke() + result = json.loads(recovery.read(self.root / 'independent-audit.json')) + self.assertEqual(result['summary_sha256'], self.summary_sha) + self.assertEqual(result['remaining'], {'container': [], 'network': [], 'volume': []}) + self.assertTrue(result['owner_process_absent'] and result['recovery_process_absent']) + with self.assertRaises(FileExistsError):self.invoke() + + def test_changed_summary_never_gains_authority(self): + (self.root / 'summary.json').write_bytes(b'{}\n') + with self.assertRaisesRegex(SystemExit, 'digest mismatch'):self.invoke() + self.assertFalse((self.root / 'independent-audit.json').exists()) + + def test_changed_watcher_receipt_refuses_even_if_it_claims_no_survivor(self): + (self.root / 'recovery-process.json').write_bytes(subject.canonical(dict(self.watcher, process={}))) + with self.assertRaisesRegex(SystemExit, 'handoff changed'):self.invoke() + + def test_each_object_and_process_survivor_refuses(self): + for kind in ('container', 'network', 'volume'): + with self.subTest(kind=kind), self.assertRaisesRegex(SystemExit, 'object survivor'): + self.invoke(objects=lambda record, k: ['synthetic'] if k == kind else []) + for pgid in (12345, 23456): + with self.subTest(pgid=pgid), self.assertRaisesRegex(SystemExit, 'process survivor'): + self.invoke(groups=lambda group: [group] if group == pgid else []) + self.assertFalse((self.root / 'independent-audit.json').exists()) + + def test_audit_does_not_follow_redirected_summary(self): + (self.root / 'summary.json').unlink() + foreign = self.root / 'foreign';foreign.write_bytes(b'preserved') + (self.root / 'summary.json').symlink_to(foreign) + with self.assertRaises(SystemExit):self.invoke() + self.assertEqual(foreign.read_bytes(), b'preserved') + + def test_late_source_drift_cannot_cancel_captured_exact_cleanup(self): + with mock.patch.object(recovery, 'validate_record', side_effect=[self.authority, SystemExit('drift')]), \ + mock.patch.object(recovery, 'stop_owner') as stop, \ + mock.patch.object(recovery, 'cleanup', return_value={'pass': True, 'errors': [], 'remaining': {}}) as cleanup, \ + self.assertRaisesRegex(SystemExit, 'recovery incomplete'): + self.invoke('recover') + stop.assert_called_once_with(self.authority) + cleanup.assert_called_once_with(self.authority) + result = json.loads(recovery.read(self.root / 'recovery-audit.json')) + self.assertFalse(result['pass']) + self.assertIn('source-or-record-drift', result['errors']) + + def test_uncertain_process_authority_still_performs_owned_daemon_cleanup(self): + with mock.patch.object(recovery, 'stop_owner', side_effect=SystemExit('foreign epoch')), \ + mock.patch.object(recovery, 'cleanup', return_value={'pass': True, 'errors': [], 'remaining': {}}) as cleanup, \ + self.assertRaisesRegex(SystemExit, 'recovery incomplete'): + self.invoke('recover') + cleanup.assert_called_once_with(self.authority) + self.assertIn('owner-process-uncertain', json.loads(recovery.read(self.root / 'recovery-audit.json'))['errors']) + + +if __name__ == '__main__':unittest.main() diff --git a/scripts/acceptance/condition-history/test_packet.py b/scripts/acceptance/condition-history/test_packet.py index fcacc43..7902626 100644 --- a/scripts/acceptance/condition-history/test_packet.py +++ b/scripts/acceptance/condition-history/test_packet.py @@ -1,7 +1,6 @@ -"""Inert source controls. Every Docker/Cargo/process launch is stubbed.""" +"""Packet guard and unchanged history controls. No Docker/Cargo/network is executed.""" import ast import copy -import importlib.util import json import os import pathlib @@ -12,69 +11,56 @@ import types import unittest from unittest import mock - -HERE = pathlib.Path(__file__).resolve().parent -spec = importlib.util.spec_from_file_location('condition_packet', HERE / 'run.py') -packet = importlib.util.module_from_spec(spec) -spec.loader.exec_module(packet) - +from test_subject import subject, fixture_subject, remove_private, REPO, HERE class Packet(unittest.TestCase): def setUp(self): - self.tmp = tempfile.TemporaryDirectory() - self.addCleanup(self.tmp.cleanup) - self.root = pathlib.Path(self.tmp.name) + self.root = pathlib.Path(tempfile.mkdtemp()).resolve() + self.addCleanup(remove_private, self.root) now = int(time.time()) - self.args = types.SimpleNamespace(work_dir=self.root / 'fresh', package_commit=packet.H, - runner_commit='a' * 40, run_token='0123456789abcdef', started_unix=now, - active_unix=now + 1740, cleanup_unix=now + 1800) - - def git(self, *args): - return {'HEAD': 'a' * 40, 'refs/tags/4.12.0^{commit}': packet.H}.get(args[-1], '') - - def test_preflight_refusals_precede_construction_and_launch(self): - cases = {'package_commit': 'b' * 40, 'runner_commit': 'not-a-commit', - 'run_token': 'foreign', 'active_unix': 0, 'started_unix': 0} - for field, value in cases.items(): - args = copy.copy(self.args) - setattr(args, field, value) - with self.subTest(field=field), mock.patch.object(packet, 'git', self.git), \ - mock.patch.object(packet, 'command') as command, \ - mock.patch.object(packet.subprocess, 'Popen') as launch, \ - self.assertRaises(SystemExit): - packet.execute(args) - command.assert_not_called() - launch.assert_not_called() - self.assertFalse(args.work_dir.exists()) + self.args = types.SimpleNamespace(mode='run', checkout=str(REPO), work_dir=str(self.root / 'fresh'), + package_commit=subject.H, runner_commit='a' * 40, run_token='0123456789abcdef', + docker_endpoint='unix:///synthetic.sock', started_unix=now, active_unix=now+1740, cleanup_unix=now+1800) + subject.ARGS = self.args + subject.CURRENT = fixture_subject(self.root) + self.packet = subject.CURRENT.module('engine') + global packet + packet = self.packet + + def git(self, checkout, *args): + return {'HEAD': 'a' * 40, 'refs/tags/4.12.0^{commit}': subject.H}.get(args[-1], '') + + def test_preflight_refusals_precede_any_side_effect(self): + for field, value in {'package_commit':'b'*40, 'run_token':'foreign', 'active_unix':0, + 'started_unix':0, 'docker_endpoint':'tcp://foreign'}.items(): + args = copy.copy(self.args);setattr(args, field, value) + with self.subTest(field=field), mock.patch.object(subject,'git',self.git), self.assertRaises(SystemExit): + subject.validate(args) + self.assertFalse(pathlib.Path(args.work_dir).exists()) def test_wrong_h_refuses_under_python_optimization(self): - result = subprocess.run([sys.executable, str(HERE / 'run.py'), 'run', '--work-dir', - str(self.args.work_dir), '--run-token', self.args.run_token, '--package-commit', 'b' * 40], - env=dict(os.environ, PYTHONOPTIMIZE='1'), capture_output=True, text=True, timeout=5) - self.assertNotEqual(result.returncode, 0) - self.assertIn('wrong fixed package source', result.stderr) - self.assertFalse(self.args.work_dir.exists()) - - def test_no_optimizable_assert_and_preflight_precedes_effects(self): - tree = ast.parse((HERE / 'run.py').read_text()) - self.assertFalse(any(isinstance(n, ast.Assert) for n in ast.walk(tree))) - execute = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'execute') - self.assertEqual(ast.unparse(execute.body[0]), 'preflight(args)') + result = subprocess.run([sys.executable,str(HERE/'run.py'),'run','--checkout',str(REPO), + '--runner-commit','a'*40,'--package-commit','b'*40], env=dict(os.environ,PYTHONOPTIMIZE='1'), + capture_output=True,text=True,timeout=5) + self.assertNotEqual(result.returncode,0) + self.assertIn('wrong fixed package source',result.stderr) + self.assertFalse(pathlib.Path(self.args.work_dir).exists()) + + def test_no_optimizable_runtime_assert(self): + for name in ('run.py','subject.py','engine.py','resources.py','recover.py'): + with self.subTest(name=name): + self.assertFalse(any(isinstance(n,ast.Assert) for n in ast.walk(ast.parse((HERE/name).read_text())))) def test_dirty_head_tag_and_existing_work_refuse(self): - for response in ('dirty', 'wrong-head', 'wrong-tag', 'existing'): - def git(*args): - if response == 'dirty' and args[0] == 'status': - return ' M source' - if response == 'wrong-head' and args[-1] == 'HEAD': - return 'b' * 40 - if response == 'wrong-tag' and args[-1].startswith('refs/tags'): - return 'b' * 40 - return self.git(*args) - if response == 'existing': - self.args.work_dir.mkdir() - with self.subTest(response=response), mock.patch.object(packet, 'git', git), self.assertRaises(SystemExit): - packet.preflight(self.args) + for response in ('dirty','wrong-head','wrong-tag','existing'): + def git(checkout,*args): + if response=='dirty' and args[0]=='status':return ' M source' + if response=='wrong-head' and args[-1]=='HEAD':return 'b'*40 + if response=='wrong-tag' and args[-1].startswith('refs/tags'):return 'b'*40 + return self.git(checkout,*args) + if response=='existing':pathlib.Path(self.args.work_dir).mkdir() + with self.subTest(response=response),mock.patch.object(subject,'git',git),self.assertRaises(SystemExit): + subject.validate(self.args) def test_exact_immutable_consumer_inputs_export_without_sdk_overlay(self): fixture = self.root / 'fixture' @@ -91,33 +77,6 @@ def test_exact_immutable_consumer_inputs_export_without_sdk_overlay(self): self.assertFalse((fixture / 'Cargo.lock').exists()) self.assertFalse((fixture / '.cargo').exists()) - def test_roles_keep_reviewed_bounds_and_credential_scopes(self): - owner = packet.native.Run(packet.PREFIX, token=self.args.run_token) - cell = packet.ConditionCell(types.SimpleNamespace(), packet.CELL, self.root, - {'executed_manifest': packet.SERVER}, self.root / 'fixture', owner) - calls = [] - def launch(_self, line, name, image, child, user, credentials): - calls.append((line, credentials)) - return name - def inspect(*args, **kwargs): - line, _ = calls[-1] - return types.SimpleNamespace(stdout=json.dumps({'id': 'synthetic', 'image': 'synthetic', - 'run': owner.token, 'owner': owner.prefix, 'pid': 1, 'mounts': 'tmpfs:/credentials;', - 'memory': int(line[line.index('--memory') + 1].removesuffix('m')) * 1024 * 1024, - 'nano_cpus': int(float(line[line.index('--cpus') + 1]) * 1_000_000_000)})) - with mock.patch.object(packet.native.Cell, 'launch', launch), mock.patch.object(packet.native, 'call', inspect): - for role, credentials in [('mysql', 'none'), ('redis', 'none'), ('bootstrap', 'server'), - ('runtime', 'both'), ('queue', 'server'), ('consumer', 'auth')]: - cell.launch(['docker', 'run', '--cpus', '1', '--memory', '1g'], cell.task + '-' + role, - packet.SERVER, ['original'], '1000:1000', credentials) - line, observed = calls[-1] - self.assertEqual(line[line.index('--cpus') + 1], packet.LIMITS[role][0]) - self.assertEqual(line[line.index('--memory') + 1], packet.LIMITS[role][1]) - self.assertEqual(observed, credentials) - cell.containers = ['held'] * 5 - with self.assertRaisesRegex(SystemExit, 'ceiling'): - cell.launch(['docker', 'run'], cell.task + '-consumer', packet.RUST, [], '1000:1000', 'auth') - def report(self): fixture = packet.native.call(['git', '-C', str(packet.REPO), 'show', packet.H + ':tests/fixtures/replay-regressions/two-member-condition-group-server-reopen.json']).stdout @@ -145,19 +104,6 @@ def test_group_comparator_rejects_lineage_order_timeout_and_false_health(self): with self.subTest(change=change), self.assertRaises((ValueError, KeyError, TypeError)): packet.comparator.check(report) - def test_foreign_recovery_record_refuses_before_docker(self): - self.args.work_dir.mkdir() - (self.args.work_dir / 'run.json').write_text(json.dumps({'owner': 'other', - 'token': self.args.run_token, 'cells': [packet.CELL]})) - with mock.patch.object(packet.native, 'recover_run') as recover, self.assertRaises(SystemExit): - packet.recover(self.args.work_dir, self.args.run_token) - recover.assert_not_called() - - def test_malformed_recovery_token_refuses_before_reading_a_record(self): - with mock.patch.object(packet.native, 'confined_child') as read, self.assertRaises(SystemExit): - packet.recover(self.root / 'absent', 'malformed') - read.assert_not_called() - def test_server_review_countercontrols(self): cases = [] report = self.report() @@ -232,5 +178,4 @@ def test_timer_terminals_require_prior_schedule_and_row_ordered_supersession(sel packet.comparator.check(self.snapshots(events)) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__":unittest.main() diff --git a/scripts/acceptance/condition-history/test_resources.py b/scripts/acceptance/condition-history/test_resources.py new file mode 100644 index 0000000..999d69c --- /dev/null +++ b/scripts/acceptance/condition-history/test_resources.py @@ -0,0 +1,170 @@ +"""Adversarial effective-cap/collision controls with a fake Docker CLI only.""" +import ast +import copy +import subprocess +import importlib.util +import json +import pathlib +import tempfile +import time +import types +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('resource_controls', HERE / 'resources.py') +resources = importlib.util.module_from_spec(spec) +spec.loader.exec_module(resources) +IMAGE = 'fixture@sha256:' + 'a' * 64 + +class Frozen: + sha256 = 'b' * 64 + def __init__(self):self.program = (HERE / 'role-supervisor.sh').read_bytes() + def verify(self):pass + def read(self, path):return self.program + +class Resources(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup) + self.root = pathlib.Path(self.tmp.name) + self.args = types.SimpleNamespace(work_dir=str(self.root / 'unused'), run_token='0123456789abcdef', + docker_endpoint='unix:///synthetic.sock', active_unix=int(time.time()) + 1740) + self.controller = resources.Docker(self.args, Frozen(), {}) + self.image = {'sha256': 'c' * 64, 'raw': {'Id': 'image-id', 'Os': 'linux', 'Architecture': 'amd64', + 'RootFS': {'Layers': []}, 'Config': {'Entrypoint': ['/entry'], 'Cmd': ['default'], + 'Env': ['PATH=/public'], 'WorkingDir': '', 'Volumes': {}}}} + self.controller.images[IMAGE] = self.image + self.controller.images['image-id'] = self.image + + def role(self, role='consumer'): + return self.controller.profile(role, IMAGE, ['explicit'], '65534:65534', 'none', self.root / 'fixture', + self.root / 'state', env={'CARGO_BUILD_JOBS': '2'} if role in ('fetch', 'build') else {}, + deadline=int(time.time()) + 500) + + def frame(self, role): + return {'Name': '/' + role['name'], 'Id': 'd' * 64, 'Image': 'image-id', 'State': {'Running': True, 'Pid': 44}, + 'Config': {'Image': IMAGE, 'User': role['user'], 'Entrypoint': ['/bin/sh'], 'Cmd': role['argv'], + 'Env': ['PATH=/public'] + [k + '=' + v for k, v in role['explicit_env'].items()], + 'Labels': self.controller.labels(role['role']), + 'WorkingDir': '/work' if role['role'] in ('fetch', 'build') else ''}, + 'HostConfig': {'NanoCpus': role['cpus'], 'Memory': role['memory'], 'MemorySwap': role['memory'], + 'PidsLimit': role['pids'], 'ReadonlyRootfs': role['readonly'], 'Init': True, 'Privileged': False, + 'CapDrop': ['ALL'], 'CapAdd': role['cap_add'], 'SecurityOpt': ['no-new-privileges'], + 'NetworkMode': role['network'], 'Tmpfs': role['tmpfs'], 'PortBindings': {}, + 'PublishAllPorts': False, 'Devices': [], 'VolumesFrom': []}, + 'Mounts': [{'Type': 'bind', 'Source': s, 'Destination': d, 'RW': rw} for s, d, rw in role['binds']] + [{'Type':'tmpfs', 'Destination':d} for d in role['tmpfs']], + 'NetworkSettings': {'Networks': {role['network']: {'Aliases': [role['alias']] if role['alias'] else []}}}} + + def network(self): + return types.SimpleNamespace(stdout=json.dumps([{'Internal': True, + 'Labels': self.controller.labels('network')}])) + + def test_actual_held_build_has_no_swap_cap(self): + repo=HERE.parents[2] + old=subprocess.check_output(['git','--no-replace-objects','-C',str(repo),'show', + 'f15509d5e6e3e8029a92a754d0a6732421a4501d:scripts/acceptance/condition-history/run.py'],text=True) + node=next(n for n in ast.parse(old).body if isinstance(n,ast.FunctionDef) and n.name=='build') + ledger=[] + namespace={'remaining':lambda args:100,'os':__import__('os'),'RUST':IMAGE, + 'command':lambda args,line,**kw:ledger.append(line) or types.SimpleNamespace(stdout='',stderr=''), + 'native':types.SimpleNamespace(remove_owned_container=lambda *a:None)} + exec(compile(ast.Module(body=[node],type_ignores=[]),'held-f155-build','exec'),namespace) + owner=types.SimpleNamespace(base='old',labels=lambda name:[],best_effort=lambda *a:None) + namespace['build'](self.args,self.root,owner,'never-executed','build') + self.assertIn('--memory',ledger[0]);self.assertNotIn('--memory-swap',ledger[0]) + new=self.controller.create_arguments(self.role('build')) + self.assertEqual(new[new.index('--memory')+1],new[new.index('--memory-swap')+1]) + + def test_every_role_has_literal_equal_swap_and_full_caps(self): + expected = {'fetch': (3 * 1024**3, 1024), 'build': (3 * 1024**3, 1024), + 'mysql': (1280 * 1024**2, 512), 'redis': (128 * 1024**2, 256), + 'bootstrap': (768 * 1024**2, 256), 'runtime': (768 * 1024**2, 256), + 'queue': (512 * 1024**2, 256), 'consumer': (768 * 1024**2, 256)} + for name, (memory, pids) in expected.items(): + role = self.role(name);argv = self.controller.create_arguments(role) + self.assertEqual(argv[argv.index('--memory') + 1], str(memory)) + self.assertEqual(argv[argv.index('--memory-swap') + 1], str(memory)) + self.assertEqual(argv[argv.index('--pids-limit') + 1], str(pids)) + self.assertIn('--init', argv);self.assertIn('no-new-privileges', argv) + with self.subTest(role=name), mock.patch.object(self.controller, 'call', return_value=self.network()): + self.assertEqual(self.controller.validate(role, self.frame(role))['memory_swap'], memory) + + def test_each_bad_effective_field_refuses_including_offline_build(self): + for name in ('consumer', 'build'): + role = self.role(name) + changes = [('HostConfig', 'MemorySwap', role['memory'] * 2), ('HostConfig', 'Memory', 0), + ('HostConfig', 'NanoCpus', 0), ('HostConfig', 'PidsLimit', -1), + ('HostConfig', 'ReadonlyRootfs', not role['readonly']), ('HostConfig', 'Init', False), + ('HostConfig', 'Privileged', True), ('HostConfig', 'CapAdd', ['SYS_ADMIN']), + ('HostConfig', 'CapDrop', []), ('HostConfig', 'SecurityOpt', []), + ('HostConfig', 'NetworkMode', 'host'), ('HostConfig', 'PortBindings', {'80/tcp': [{}]}), + ('HostConfig', 'Tmpfs', {}), ('Config', 'User', '0:0'), ('Config', 'Cmd', ['foreign']), + ('Config', 'Entrypoint', ['foreign']), ('Config', 'Env', ['CARGO_BUILD_JOBS=99']), + ('Config', 'Image', 'foreign'), ('Config', 'WorkingDir', '/foreign'), ('Config', 'Labels', {}), + ('State', 'Running', False), ('State', 'Pid', 0)] + for section, key, value in changes: + frame = copy.deepcopy(self.frame(role));frame[section][key] = value + with self.subTest(role=name, field=key), mock.patch.object(self.controller, 'call', return_value=self.network()), self.assertRaises(SystemExit): + self.controller.validate(role, frame) + + def test_internal_network_and_mount_authority_are_effective_checks(self): + role = self.role() + frame = self.frame(role) + with mock.patch.object(self.controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([ + {'Internal': False, 'Labels': self.controller.labels('network')}]))), self.assertRaises(SystemExit): + self.controller.validate(role, frame) + for mutate in (lambda f: f['Mounts'].append({'Type': 'volume', 'Destination': '/extra'}), + lambda f: f['Mounts'].pop(), + lambda f: f.update(Name='/foreign'), + lambda f: f['Mounts'][0].update(RW=True), + lambda f: f['Mounts'][0].update(Source='/foreign'), + lambda f: f['NetworkSettings'].update(Networks={'foreign': {}})): + frame = copy.deepcopy(self.frame(role));mutate(frame) + with self.subTest(mutate=mutate), mock.patch.object(self.controller, 'call', return_value=self.network()), self.assertRaises(SystemExit): + self.controller.validate(role, frame) + + def test_no_child_or_credential_gate_opens_before_readback(self): + role = self.role();frame = self.frame(role);frame['HostConfig']['MemorySwap'] *= 2 + ledger = [] + def cli(line, **kwargs): + ledger.append(line) + return types.SimpleNamespace(stdout=json.dumps([frame]) if line[:2] == ['docker', 'inspect'] else '') + with mock.patch.object(self.controller, 'call', cli), self.assertRaises(SystemExit): + self.controller.start(role, token='synthetic-only', key='synthetic-only') + self.assertFalse(any(line[:2] == ['docker', 'exec'] for line in ledger)) + self.assertEqual(self.controller.receipts, []) + + def test_compile_input_and_replay_input_mounts_are_readonly(self): + build = self.role('build');consumer = self.role('consumer') + self.assertIn((str(self.root / 'fixture'), '/work', False), build['binds']) + self.assertIn((str(self.root / 'fixture'), '/work', False), consumer['binds']) + self.assertEqual({m[1] for m in build['binds'] if m[2]}, {'/work/.cargo', '/work/target'}) + self.assertEqual(self.role('fetch')['network'], 'bridge') + self.assertEqual(build['network'], 'none') + + def test_wrong_platform_and_image_content_refuse(self): + controller = resources.Docker(self.args, Frozen(), {}) + raw = copy.deepcopy(self.image['raw']);raw['Architecture'] = 'arm64' + with mock.patch.object(controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([raw]))), self.assertRaises(SystemExit): + controller.image(IMAGE) + role = self.role();self.controller.images['image-id'] = dict(self.image, sha256='foreign') + with self.assertRaises(SystemExit):self.controller.validate(role, self.frame(role)) + + def test_any_owner_or_exact_name_collision_precedes_mutation(self): + for collide in ('container', 'network', 'volume', 'foreign-name', 'uncertain'): + ledger = [] + def cli(args, line, **kwargs): + ledger.append(line) + family = 'container' if line[1] == 'ps' else 'network' if line[1:3] == ['network', 'ls'] else 'volume' + if 'inspect' not in line: + return types.SimpleNamespace(stdout='existing' if family == collide else '', returncode=0, stderr='') + return types.SimpleNamespace(stdout='existing' if collide == 'foreign-name' else '', + returncode=0 if collide == 'foreign-name' else 1, + stderr='daemon unavailable' if collide == 'uncertain' else 'No such object') + with self.subTest(collide=collide), mock.patch.object(resources, 'call', cli), self.assertRaises(SystemExit): + resources.preflight_collisions(self.args) + self.assertFalse(pathlib.Path(self.args.work_dir).exists()) + self.assertFalse(any(x in line for line in ledger for x in ('create', 'pull', 'start', 'rm'))) + + +if __name__ == '__main__':unittest.main() diff --git a/scripts/acceptance/condition-history/test_subject.py b/scripts/acceptance/condition-history/test_subject.py new file mode 100644 index 0000000..c4934e2 --- /dev/null +++ b/scripts/acceptance/condition-history/test_subject.py @@ -0,0 +1,241 @@ +"""Closed subject/record controls. No Docker, Cargo or network is executed.""" +import ast +import copy +import hashlib +import importlib.util +import json +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +REPO = HERE.parents[2] + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + +subject = load('sealed_subject', HERE / 'subject.py') +recovery = load('subject_recovery_tests', HERE / 'recover.py') + + +def fixture_subject(root): + data = {p: (REPO / p).read_bytes() for p in subject.RUNTIME_FILES} + files = {p: {'git_blob': 'a' * 40, 'mode': '100644', 'bytes': len(v), 'sha256': subject.digest(v)} + for p, v in data.items()} + value = {'schema': 'durable-workflow.condition-subject/v1', 'runner_commit': 'a' * 40, + 'package_commit': subject.H, 'files': files} + sealed = subject.Subject(value, data, root / 'snapshot') + sealed.materialize() + return sealed + + +def remove_private(root): + for parent, dirs, files in os.walk(root, followlinks=False): + os.chmod(parent, 0o700) + for name in dirs: + p = pathlib.Path(parent) / name + if not p.is_symlink():p.chmod(0o700) + shutil.rmtree(root) + + +class SubjectControls(unittest.TestCase): + def setUp(self): + self.root = pathlib.Path(tempfile.mkdtemp()).resolve() + self.addCleanup(remove_private, self.root) + self.sealed = fixture_subject(self.root) + + def old_function(self, name, namespace): + old = subprocess.check_output(['git', '--no-replace-objects', '-C', str(REPO), 'show', + 'f15509d5e6e3e8029a92a754d0a6732421a4501d:scripts/acceptance/condition-history/run.py'], text=True) + node = next(n for n in ast.parse(old).body if isinstance(n, ast.FunctionDef) and n.name == name) + exec(compile(ast.Module(body=[node], type_ignores=[]), 'held-f155-function', 'exec'), namespace) + return namespace[name] + + def test_actual_held_export_function_admits_live_edit_restore(self): + live = self.root / 'live';(live / 'fixture').mkdir(parents=True) + source = live / 'fixture/marker';source.write_bytes(b'reviewed') + old_export = self.old_function('export_consumer', {'shutil':shutil,'HERE':live,'INPUTS':{}}) + source.write_bytes(b'unreviewed') + old_export(self.root / 'old-export') + source.write_bytes(b'reviewed') + self.assertEqual((self.root / 'old-export/marker').read_bytes(), b'unreviewed') + output = self.root / 'new-export' + self.sealed.export(subject.PREFIX+'fixture/', output) + self.assertEqual((output/'tests/conditions.rs').read_bytes(), + self.sealed.captured[subject.PREFIX+'fixture/tests/conditions.rs']) + + def test_actual_held_recovery_accepts_missing_source_and_deadline_authority(self): + path = self.root/'old-record.json' + path.write_text(json.dumps({'owner':recovery.PREFIX,'token':'0123456789abcdef','cells':[recovery.CELL]})) + calls=[] + native=types.SimpleNamespace(confined_child=lambda *a:path,Run=lambda *a,**k:None, + recover_run=lambda *a:calls.append(a) or {'pass':True}) + old_recover=self.old_function('recover',{'json':json,'re':__import__('re'),'native':native, + 'PREFIX':recovery.PREFIX,'CELL':recovery.CELL,'require':subject.require}) + self.assertEqual(old_recover(self.root,'0123456789abcdef'),{'pass':True}) + self.assertEqual(len(calls),1) + incomplete=json.loads(path.read_text()) + with self.assertRaises(SystemExit):self.validate(incomplete) + + def test_actual_held_summary_statement_overwrites_an_existing_handoff(self): + old = subprocess.check_output(['git','--no-replace-objects','-C',str(REPO),'show', + 'f15509d5e6e3e8029a92a754d0a6732421a4501d:scripts/acceptance/condition-history/run.py'],text=True) + node=next(n for n in ast.walk(ast.parse(old)) if isinstance(n,ast.Call) + and isinstance(n.func,ast.Attribute) and n.func.attr=='write_bytes' + and isinstance(n.func.value,ast.BinOp) and isinstance(n.func.value.right,ast.Constant) + and n.func.value.right.value=='summary.json') + path=self.root/'summary.json';path.write_bytes(b'prior-handoff') + eval(compile(ast.Expression(node),'held-f155-handoff','eval'), + {'args':types.SimpleNamespace(work_dir=self.root),'clean':b'overwritten'}) + self.assertEqual(path.read_bytes(),b'overwritten'+b'\n') + with self.assertRaises(FileExistsError):subject.exclusive(path,b'new') + self.assertEqual(path.read_bytes(),b'overwritten'+b'\n') + + def test_post_preflight_live_replacement_and_restore_never_changes_export(self): + live = self.root / 'live.rs' + path = subject.PREFIX + 'fixture/tests/conditions.rs' + reviewed = self.sealed.captured[path] + live.write_bytes(reviewed) + # Old source copy reads the changed live path and admits it. + live.write_bytes(b'unreviewed replacement') + old_copy = self.root / 'old-copy.rs' + shutil.copyfile(live, old_copy) + live.write_bytes(reviewed) + self.assertNotEqual(old_copy.read_bytes(), reviewed) + output = self.root / 'new-copy' + self.sealed.export(subject.PREFIX + 'fixture/', output) + self.assertEqual((output / 'tests/conditions.rs').read_bytes(), reviewed) + + def test_source_replacement_is_detected_before_module_use(self): + p = self.sealed.root / (subject.PREFIX + 'engine.py') + p.chmod(0o600) + p.write_bytes(b'raise SystemExit("changed")');p.chmod(0o400) + with self.assertRaisesRegex(SystemExit, 'changed'): + self.sealed.module('engine') + + def test_extra_importable_missing_link_and_nonregular_members_refuse(self): + for kind in ('extra', 'empty-dir', 'missing', 'symlink', 'hardlink', 'fifo'): + root = self.root / kind + root.mkdir() + sealed = fixture_subject(root) + target = sealed.root / (subject.PREFIX + 'engine.py') + target.parent.chmod(0o700) + if kind == 'extra': + (target.parent / 'sitecustomize.py').write_text('raise RuntimeError()') + elif kind == 'empty-dir':(target.parent / 'unreviewed').mkdir() + elif kind == 'missing':target.unlink() + elif kind == 'symlink': + target.unlink();target.symlink_to(self.root / 'foreign') + elif kind == 'hardlink':os.link(target, self.root / 'alias') + else: + target.unlink();os.mkfifo(target) + with self.subTest(kind=kind), self.assertRaises((SystemExit, OSError)): + sealed.verify() + + def test_captured_recovery_is_not_reopened_from_a_mutable_path(self): + path = subject.PREFIX + 'recover.py' + code = self.sealed.read(path) + command = [sys.executable, '-I', '-S', '-c', code.decode()] + target = self.sealed.root / path + target.chmod(0o600);target.write_bytes(b'print("foreign")') + self.assertEqual(subject.digest(command[-1].encode()), self.sealed.value['files'][path]['sha256']) + self.assertNotEqual(command[-1].encode(), target.read_bytes()) + with self.assertRaises(SystemExit):self.sealed.read(path) + + def test_closed_loader_refuses_any_other_module(self): + with self.assertRaises(SystemExit):self.sealed.module('sitecustomize') + + def test_exclusive_summary_handoff_rejects_existing_file_and_redirect(self): + data = b'{"checks_outcome":"pass"}\n' + path = self.root / 'summary.json' + sha = subject.exclusive(path, data) + self.assertEqual(sha, subject.digest(data)) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + with self.assertRaises(FileExistsError):subject.exclusive(path, b'changed') + foreign = self.root / 'foreign';foreign.write_bytes(b'preserved') + link = self.root / 'summary-link';link.symlink_to(foreign) + with self.assertRaises(FileExistsError):subject.exclusive(link, b'changed') + self.assertEqual(foreign.read_bytes(), b'preserved') + + def record(self): + now = int(time.time()) + return {'schema': 'durable-workflow.condition-run/v2', 'owner': recovery.PREFIX, + 'token': '0123456789abcdef', 'cells': [recovery.CELL], 'runner_commit': 'a' * 40, + 'package_commit': subject.H, 'manifest_sha256': self.sealed.sha256, + 'recovery_sha256': self.sealed.value['files'][subject.PREFIX + 'recover.py']['sha256'], + 'started_unix': now, 'active_unix': now + 1740, 'cleanup_unix': now + 1800, + 'work_dir': str(self.root), 'docker_endpoint': 'unix:///synthetic.sock', + 'uid': os.getuid(), 'gid': os.getgid(), + 'owner_process': {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'started': 'synthetic epoch'}} + + def validate(self, record, expected_sha=None): + path = self.root / 'run.json' + if path.exists():path.unlink() + raw = subject.canonical(record) + subject.exclusive(path, raw) + return recovery.validate_record(self.root, expected_sha or subject.digest(raw), 'a' * 40, + self.sealed.sha256, self.record()['recovery_sha256'], '0123456789abcdef') + + def test_complete_recovery_authority(self): + record = self.record() + self.assertEqual(self.validate(record), record) + for field, value in [('extra', 1), ('package_commit', 'b' * 40), ('runner_commit', 'b' * 40), + ('manifest_sha256', 'b' * 64), ('recovery_sha256', 'b' * 64), ('cleanup_unix', 0), + ('started_unix', True), ('owner', 'other'), ('work_dir', str(self.root / 'other'))]: + bad = copy.deepcopy(record);bad[field] = value + with self.subTest(field=field), self.assertRaises(SystemExit):self.validate(bad) + missing = dict(record);del missing['active_unix'] + with self.assertRaises(SystemExit):self.validate(missing) + + def test_edit_of_record_does_not_gain_authority_by_recomputing_other_fields(self): + record = self.record();expected = subject.digest(subject.canonical(record)) + record['token'] = 'fedcba9876543210' + with self.assertRaises(SystemExit):self.validate(record, expected) + + def test_record_links_and_wrong_modes_refuse(self): + record = self.record();self.validate(record) + path = self.root / 'run.json' + path.chmod(0o644) + with self.assertRaises(SystemExit): + recovery.validate_record(self.root, subject.digest(subject.canonical(record)), 'a' * 40, + self.sealed.sha256, record['recovery_sha256'], record['token']) + path.chmod(0o600);os.link(path, self.root / 'record-alias') + with self.assertRaises(SystemExit): + recovery.validate_record(self.root, subject.digest(subject.canonical(record)), 'a' * 40, + self.sealed.sha256, record['recovery_sha256'], record['token']) + + def test_foreign_owner_or_renamed_object_is_never_removed(self): + record = self.record() + base = recovery.PREFIX + '-' + record['token'] + '-' + recovery.CELL + good = {'labels': {recovery.RUN: record['token'], recovery.OWNER: recovery.PREFIX, + recovery.SUBJECT: record['manifest_sha256'], recovery.ROLE: 'consumer'}, + 'name': '/' + base + '-consumer', 'id': 'a' * 64} + for field, value in [('owner', 'foreign'), ('subject', 'b' * 64), ('name', '/foreign')]: + frame = copy.deepcopy(good) + if field == 'name':frame['name'] = value + else:frame['labels'][recovery.OWNER if field == 'owner' else recovery.SUBJECT] = value + with self.subTest(field=field), mock.patch.object(recovery, 'docker', + return_value=types.SimpleNamespace(returncode=0, stdout=json.dumps(frame))) as cli, self.assertRaises(SystemExit): + recovery.owned(record, 'container', 'candidate') + self.assertEqual(cli.call_count, 1) + + def test_changed_pid_epoch_never_kills_a_foreign_group(self): + record = self.record();changed = dict(record['owner_process'], started='other epoch') + with mock.patch.object(recovery, 'process_identity', return_value=changed), \ + mock.patch.object(recovery.os, 'killpg') as kill, self.assertRaises(SystemExit): + recovery.stop_owner(record) + kill.assert_not_called() + + +if __name__ == '__main__':unittest.main() From f183951488edacd3fba4c30861a57c2947977c01 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:10:33 +0000 Subject: [PATCH 04/14] test: qualify recovery readiness and independent audit handoffs --- .github/workflows/ci.yml | 2 + .../acceptance/condition-history/engine.py | 38 ++++++++----- .../acceptance/condition-history/recover.py | 6 ++- .../acceptance/condition-history/resources.py | 6 ++- .../condition-history/test_audit.py | 15 +++++- .../condition-history/test_packet.py | 54 +++++++++++++++++++ .../condition-history/test_resources.py | 22 +++++++- .../condition-history/test_subject.py | 6 ++- 8 files changed, 128 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92bc9f7..88294fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,6 +157,8 @@ jobs: run: | python scripts/acceptance/installed-native/test_run.py python -m unittest discover -s scripts/acceptance/installed-native -p test_cell_lifetime.py + - name: Test the condition-history runner without Docker or Cargo + run: python -B -m unittest discover -s scripts/acceptance/condition-history -p 'test_*.py' - name: Check committed Cargo locks for known advisories run: python scripts/ci/check-dependency-advisories.py diff --git a/scripts/acceptance/condition-history/engine.py b/scripts/acceptance/condition-history/engine.py index 70a71d8..8d1c59e 100644 --- a/scripts/acceptance/condition-history/engine.py +++ b/scripts/acceptance/condition-history/engine.py @@ -216,15 +216,7 @@ def execute(self, args): return outcome -def execute(args): - # The captured bootstrap has already verified R/H/manifest and refused all - # collisions before creating this root or contacting a registry. - record = recovery.validate_record(args.work_dir, subject.RECORD_SHA, args.runner_commit, - subject.CURRENT.sha256, subject.RECORD['recovery_sha256'], args.run_token) - owner = native.Run(PREFIX, token=args.run_token) - owner.base = PREFIX + '-' + args.run_token - controller = resources.Docker(args, subject.CURRENT, record) - native.call = lambda line, **kwargs: command(args, line, **kwargs) +def start_recovery(args, record): program = subject.CURRENT.read(subject.PREFIX + 'recover.py').decode() require(subject.digest(program.encode()) == record['recovery_sha256'], 'captured recovery source mismatch') recovery_argv = [sys.executable, '-I', '-S', '-c', program, 'recover', '--work-dir', args.work_dir, @@ -233,11 +225,11 @@ def execute(args): '--at', str(args.active_unix)] watchdog = subprocess.Popen(recovery_argv, start_new_session=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=resources.environment(args)) - process = recovery.process_identity(watchdog.pid) - require(process is not None, 'recovery did not start') - recovery_process_sha = subject.exclusive(pathlib.Path(args.work_dir) / 'recovery-process.json', subject.canonical({ - 'source_sha256': record['recovery_sha256'], 'process': process})) try: + process = recovery.process_identity(watchdog.pid) + require(process is not None, 'recovery did not start') + recovery_process_sha = subject.exclusive(pathlib.Path(args.work_dir) / 'recovery-process.json', subject.canonical({ + 'source_sha256': record['recovery_sha256'], 'process': process})) ready_path = pathlib.Path(args.work_dir) / 'recovery-ready.json' until = min(time.time() + 5, args.active_unix - 10) while not ready_path.exists(): @@ -248,8 +240,26 @@ def execute(args): 'recovery_sha256': record['recovery_sha256'], 'process': process}, 'wrong recovery-ready authority') except BaseException: if watchdog.poll() is None: - watchdog.terminate();watchdog.wait(timeout=5) + watchdog.terminate() + try: + watchdog.wait(timeout=5) + except subprocess.TimeoutExpired: + watchdog.kill() + watchdog.wait(timeout=5) raise + return watchdog, recovery_process_sha + + +def execute(args): + # The captured bootstrap has already verified R/H/manifest and refused all + # collisions before creating this root or contacting a registry. + record = recovery.validate_record(args.work_dir, subject.RECORD_SHA, args.runner_commit, + subject.CURRENT.sha256, subject.RECORD['recovery_sha256'], args.run_token) + owner = native.Run(PREFIX, token=args.run_token) + owner.base = PREFIX + '-' + args.run_token + controller = resources.Docker(args, subject.CURRENT, record) + native.call = lambda line, **kwargs: command(args, line, **kwargs) + watchdog, recovery_process_sha = start_recovery(args, record) result = {'outcome': 'fail', 'checks_outcome': 'fail', 'record_sha256': subject.RECORD_SHA, 'runner_commit': args.runner_commit, 'package_commit': H, 'manifest_sha256': subject.CURRENT.sha256, 'recovery_process_sha256': recovery_process_sha} diff --git a/scripts/acceptance/condition-history/recover.py b/scripts/acceptance/condition-history/recover.py index d5b0c1b..e1d3456 100644 --- a/scripts/acceptance/condition-history/recover.py +++ b/scripts/acceptance/condition-history/recover.py @@ -330,7 +330,11 @@ def main(argv=None): and process['pid'] > 1 and process['pid'] == process['pgid'] == process['sid'] and isinstance(process['started'], str) and process['started'], 'wrong recovery-process identity') require(not group_members(watcher['process']['pgid']), 'independent recovery process survivor') - require(summary.get('checks_outcome') == 'pass', 'source checks did not pass') + require(summary.get('checks_outcome') == 'pass' + and summary.get('outcome') == 'pending_independent_audit' + and summary.get('cleanup') == {'pass': True, 'errors': [], + 'remaining': {'container': [], 'network': [], 'volume': []}} + and summary.get('cleanup_errors') == [], 'owner checks or cleanup did not pass') result = {'outcome': 'pass', 'record_sha256': args.record_sha256, 'manifest_sha256': args.manifest_sha256, 'summary_sha256': args.summary_sha256, 'remaining': remaining, 'owner_process_absent': True, 'recovery_process_absent': True} diff --git a/scripts/acceptance/condition-history/resources.py b/scripts/acceptance/condition-history/resources.py index 4cec97c..df833ad 100644 --- a/scripts/acceptance/condition-history/resources.py +++ b/scripts/acceptance/condition-history/resources.py @@ -174,8 +174,10 @@ def validate(self, spec, data): require(caps(host['CapDrop']) == ['ALL'] and caps(host['CapAdd']) == caps(spec['cap_add']), 'wrong capability set') require(sorted(v.removesuffix(':true') for v in host['SecurityOpt']) == ['no-new-privileges'], 'wrong security options') require(host['NetworkMode'] == spec['network'] and not host.get('PortBindings') and not host.get('PublishAllPorts') - and not host.get('Devices') and not host.get('VolumesFrom') - and host.get('PidMode', '') != 'host' and host.get('IpcMode', 'private') != 'host', 'wrong effective network/device sharing') + and not host.get('Devices') and not host.get('DeviceRequests') and not host.get('DeviceCgroupRules') + and not host.get('VolumesFrom') and host.get('PidMode', '') == '' + and host.get('IpcMode', 'private') == 'private' and host.get('UTSMode', '') == '', + 'wrong effective network/device sharing') if spec['network'] == self.task: network = json.loads(self.call(['docker', 'network', 'inspect', self.task]).stdout)[0] require(network['Internal'] is True and all(network['Labels'].get(k) == v for k, v in self.labels('network').items()), 'wrong internal network authority') diff --git a/scripts/acceptance/condition-history/test_audit.py b/scripts/acceptance/condition-history/test_audit.py index 7f5ba72..e981639 100644 --- a/scripts/acceptance/condition-history/test_audit.py +++ b/scripts/acceptance/condition-history/test_audit.py @@ -30,7 +30,9 @@ def setUp(self): self.summary = {'checks_outcome': 'pass', 'outcome': 'pending_independent_audit', 'record_sha256': self.record_sha, 'runner_commit': self.authority['runner_commit'], 'package_commit': subject.H, 'manifest_sha256': self.sealed.sha256, - 'recovery_process_sha256': watcher_sha} + 'recovery_process_sha256': watcher_sha, 'cleanup_errors': [], + 'cleanup': {'pass': True, 'errors': [], + 'remaining': {'container': [], 'network': [], 'volume': []}}} self.summary_sha = subject.exclusive(self.root / 'summary.json', subject.canonical(self.summary)) def args(self, mode='audit'): @@ -81,6 +83,17 @@ def test_audit_does_not_follow_redirected_summary(self): with self.assertRaises(SystemExit):self.invoke() self.assertEqual(foreign.read_bytes(), b'preserved') + def test_failed_or_incomplete_owner_result_never_gains_audit_pass(self): + for field, value in [('outcome', 'fail'), ('checks_outcome', 'fail'), + ('cleanup', {}), ('cleanup_errors', ['uncertain'])]: + bad = dict(self.summary, **{field: value}) + raw = subject.canonical(bad) + (self.root / 'summary.json').write_bytes(raw) + self.summary_sha = subject.digest(raw) + with self.subTest(field=field), self.assertRaisesRegex(SystemExit, 'owner checks or cleanup'): + self.invoke() + self.assertFalse((self.root / 'independent-audit.json').exists()) + def test_late_source_drift_cannot_cancel_captured_exact_cleanup(self): with mock.patch.object(recovery, 'validate_record', side_effect=[self.authority, SystemExit('drift')]), \ mock.patch.object(recovery, 'stop_owner') as stop, \ diff --git a/scripts/acceptance/condition-history/test_packet.py b/scripts/acceptance/condition-history/test_packet.py index 7902626..7644b33 100644 --- a/scripts/acceptance/condition-history/test_packet.py +++ b/scripts/acceptance/condition-history/test_packet.py @@ -51,6 +51,60 @@ def test_no_optimizable_runtime_assert(self): with self.subTest(name=name): self.assertFalse(any(isinstance(n,ast.Assert) for n in ast.walk(ast.parse((HERE/name).read_text())))) + def recovery_start(self, changed=None, process_missing=False, exited=False): + pathlib.Path(self.args.work_dir).mkdir() + subject.RECORD_SHA = 'c' * 64 + process = {'pid': 23456, 'pgid': 23456, 'sid': 23456, 'started': 'synthetic watcher'} + recovery_sha = subject.CURRENT.value['files'][subject.PREFIX + 'recover.py']['sha256'] + ready = {'record_sha256': subject.RECORD_SHA, 'manifest_sha256': subject.CURRENT.sha256, + 'recovery_sha256': recovery_sha, 'process': process} + ready.update(changed or {}) + if not exited: + subject.exclusive(pathlib.Path(self.args.work_dir) / 'recovery-ready.json', subject.canonical(ready)) + watcher = mock.Mock(pid=process['pid']) + watcher.poll.return_value = 1 if exited else None + with mock.patch.object(packet.subprocess, 'Popen', return_value=watcher) as launch, \ + mock.patch.object(packet.recovery, 'process_identity', return_value=None if process_missing else process), \ + mock.patch.object(packet, 'registry_identity', side_effect=AssertionError('no registry before readiness')), \ + mock.patch.object(packet.resources, 'call', side_effect=AssertionError('no daemon')): + if changed or process_missing or exited: + with self.assertRaises(SystemExit): + packet.start_recovery(self.args, {'recovery_sha256': recovery_sha}) + else: + actual, receipt_sha = packet.start_recovery(self.args, {'recovery_sha256': recovery_sha}) + self.assertIs(actual, watcher) + receipt = (pathlib.Path(self.args.work_dir) / 'recovery-process.json').read_bytes() + self.assertEqual(subject.digest(receipt), receipt_sha) + self.assertEqual(json.loads(receipt), {'source_sha256': recovery_sha, 'process': process}) + argv = launch.call_args.args[0] + self.assertEqual(argv[:4], [sys.executable, '-I', '-S', '-c']) + self.assertEqual(subject.digest(argv[4].encode()), recovery_sha) + self.assertEqual(argv[-2:], ['--at', str(self.args.active_unix)]) + self.assertTrue(launch.call_args.kwargs['start_new_session']) + return watcher + + def test_captured_recovery_requires_exact_ready_receipt(self): + watcher = self.recovery_start() + watcher.terminate.assert_not_called() + + def test_wrong_recovery_ready_refuses_before_registry_or_daemon(self): + for key in ('record_sha256', 'manifest_sha256', 'recovery_sha256', 'process'): + with self.subTest(key=key): + watcher = self.recovery_start({key: 'foreign'}) + watcher.terminate.assert_called_once() + watcher.wait.assert_called_once_with(timeout=5) + import shutil + shutil.rmtree(self.args.work_dir) + + def test_missing_recovery_identity_stops_the_started_child(self): + watcher = self.recovery_start(process_missing=True) + watcher.terminate.assert_called_once() + watcher.wait.assert_called_once_with(timeout=5) + + def test_exited_recovery_refuses_without_launching_any_role(self): + watcher = self.recovery_start(exited=True) + watcher.terminate.assert_not_called() + def test_dirty_head_tag_and_existing_work_refuse(self): for response in ('dirty','wrong-head','wrong-tag','existing'): def git(checkout,*args): diff --git a/scripts/acceptance/condition-history/test_resources.py b/scripts/acceptance/condition-history/test_resources.py index 999d69c..2747aee 100644 --- a/scripts/acceptance/condition-history/test_resources.py +++ b/scripts/acceptance/condition-history/test_resources.py @@ -62,7 +62,7 @@ def network(self): def test_actual_held_build_has_no_swap_cap(self): repo=HERE.parents[2] old=subprocess.check_output(['git','--no-replace-objects','-C',str(repo),'show', - 'f15509d5e6e3e8029a92a754d0a6732421a4501d:scripts/acceptance/condition-history/run.py'],text=True) + '67a37f7f0e42ba63ee4e68cae3090a9ba73e1cc2:scripts/acceptance/condition-history/run.py'],text=True) node=next(n for n in ast.parse(old).body if isinstance(n,ast.FunctionDef) and n.name=='build') ledger=[] namespace={'remaining':lambda args:100,'os':__import__('os'),'RUST':IMAGE, @@ -98,6 +98,9 @@ def test_each_bad_effective_field_refuses_including_offline_build(self): ('HostConfig', 'Privileged', True), ('HostConfig', 'CapAdd', ['SYS_ADMIN']), ('HostConfig', 'CapDrop', []), ('HostConfig', 'SecurityOpt', []), ('HostConfig', 'NetworkMode', 'host'), ('HostConfig', 'PortBindings', {'80/tcp': [{}]}), + ('HostConfig', 'PidMode', 'container:foreign'), ('HostConfig', 'IpcMode', 'container:foreign'), + ('HostConfig', 'UTSMode', 'host'), ('HostConfig', 'DeviceRequests', [{'Count': -1}]), + ('HostConfig', 'DeviceCgroupRules', ['a *:* rwm']), ('HostConfig', 'Tmpfs', {}), ('Config', 'User', '0:0'), ('Config', 'Cmd', ['foreign']), ('Config', 'Entrypoint', ['foreign']), ('Config', 'Env', ['CARGO_BUILD_JOBS=99']), ('Config', 'Image', 'foreign'), ('Config', 'WorkingDir', '/foreign'), ('Config', 'Labels', {}), @@ -134,6 +137,23 @@ def cli(line, **kwargs): self.assertFalse(any(line[:2] == ['docker', 'exec'] for line in ledger)) self.assertEqual(self.controller.receipts, []) + def test_valid_readback_precedes_private_credential_release(self): + role = self.role();frame = self.frame(role) + ledger = [] + def cli(line, **kwargs): + ledger.append((line, kwargs)) + if line[:3] == ['docker', 'network', 'inspect']: + return self.network() + if line[:2] == ['docker', 'inspect']: + return types.SimpleNamespace(stdout=json.dumps([frame])) + return types.SimpleNamespace(stdout='') + with mock.patch.object(self.controller, 'call', cli): + self.assertEqual(self.controller.start(role, token='synthetic-only'), role['name']) + self.assertEqual(len(self.controller.receipts), 1) + self.assertEqual(ledger[-1][0][:3], ['docker', 'exec', '-i']) + self.assertEqual(ledger[-1][1]['input'], 'synthetic-only\n\n') + self.assertLess(next(i for i, (line, _) in enumerate(ledger) if line[:2] == ['docker', 'inspect']), len(ledger) - 1) + def test_compile_input_and_replay_input_mounts_are_readonly(self): build = self.role('build');consumer = self.role('consumer') self.assertIn((str(self.root / 'fixture'), '/work', False), build['binds']) diff --git a/scripts/acceptance/condition-history/test_subject.py b/scripts/acceptance/condition-history/test_subject.py index c4934e2..f6a6eeb 100644 --- a/scripts/acceptance/condition-history/test_subject.py +++ b/scripts/acceptance/condition-history/test_subject.py @@ -56,8 +56,10 @@ def setUp(self): self.sealed = fixture_subject(self.root) def old_function(self, name, namespace): + # This ancestor preserves the held f155 run.py blob and remains reachable + # in ordinary clones after the original review branches are removed. old = subprocess.check_output(['git', '--no-replace-objects', '-C', str(REPO), 'show', - 'f15509d5e6e3e8029a92a754d0a6732421a4501d:scripts/acceptance/condition-history/run.py'], text=True) + '67a37f7f0e42ba63ee4e68cae3090a9ba73e1cc2:scripts/acceptance/condition-history/run.py'], text=True) node = next(n for n in ast.parse(old).body if isinstance(n, ast.FunctionDef) and n.name == name) exec(compile(ast.Module(body=[node], type_ignores=[]), 'held-f155-function', 'exec'), namespace) return namespace[name] @@ -90,7 +92,7 @@ def test_actual_held_recovery_accepts_missing_source_and_deadline_authority(self def test_actual_held_summary_statement_overwrites_an_existing_handoff(self): old = subprocess.check_output(['git','--no-replace-objects','-C',str(REPO),'show', - 'f15509d5e6e3e8029a92a754d0a6732421a4501d:scripts/acceptance/condition-history/run.py'],text=True) + '67a37f7f0e42ba63ee4e68cae3090a9ba73e1cc2:scripts/acceptance/condition-history/run.py'],text=True) node=next(n for n in ast.walk(ast.parse(old)) if isinstance(n,ast.Call) and isinstance(n.func,ast.Attribute) and n.func.attr=='write_bytes' and isinstance(n.func.value,ast.BinOp) and isinstance(n.func.value.right,ast.Constant) From e309116cb0dd79e415ae6a25b809515354cea026 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:13:36 +0000 Subject: [PATCH 05/14] test: isolate the expected failure in runner controls --- scripts/acceptance/condition-history/test_packet.py | 3 ++- scripts/acceptance/condition-history/test_resources.py | 3 ++- scripts/acceptance/condition-history/test_subject.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/acceptance/condition-history/test_packet.py b/scripts/acceptance/condition-history/test_packet.py index 7644b33..4ccbeaa 100644 --- a/scripts/acceptance/condition-history/test_packet.py +++ b/scripts/acceptance/condition-history/test_packet.py @@ -228,8 +228,9 @@ def test_timer_terminals_require_prior_schedule_and_row_ordered_supersession(sel events.insert(2 if label == 'before-reopen' else -1, fired) if label == 'duplicate-fire': events.insert(-1, copy.deepcopy(fired)) + report = self.snapshots(events) with self.subTest(label=label), self.assertRaises(ValueError): - packet.comparator.check(self.snapshots(events)) + packet.comparator.check(report) if __name__ == "__main__":unittest.main() diff --git a/scripts/acceptance/condition-history/test_resources.py b/scripts/acceptance/condition-history/test_resources.py index 2747aee..688029c 100644 --- a/scripts/acceptance/condition-history/test_resources.py +++ b/scripts/acceptance/condition-history/test_resources.py @@ -168,7 +168,8 @@ def test_wrong_platform_and_image_content_refuse(self): with mock.patch.object(controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([raw]))), self.assertRaises(SystemExit): controller.image(IMAGE) role = self.role();self.controller.images['image-id'] = dict(self.image, sha256='foreign') - with self.assertRaises(SystemExit):self.controller.validate(role, self.frame(role)) + frame = self.frame(role) + with self.assertRaises(SystemExit):self.controller.validate(role, frame) def test_any_owner_or_exact_name_collision_precedes_mutation(self): for collide in ('container', 'network', 'volume', 'foreign-name', 'uncertain'): diff --git a/scripts/acceptance/condition-history/test_subject.py b/scripts/acceptance/condition-history/test_subject.py index f6a6eeb..c6a5f2e 100644 --- a/scripts/acceptance/condition-history/test_subject.py +++ b/scripts/acceptance/condition-history/test_subject.py @@ -207,14 +207,15 @@ def test_edit_of_record_does_not_gain_authority_by_recomputing_other_fields(self def test_record_links_and_wrong_modes_refuse(self): record = self.record();self.validate(record) + record_sha = subject.digest(subject.canonical(record)) path = self.root / 'run.json' path.chmod(0o644) with self.assertRaises(SystemExit): - recovery.validate_record(self.root, subject.digest(subject.canonical(record)), 'a' * 40, + recovery.validate_record(self.root, record_sha, 'a' * 40, self.sealed.sha256, record['recovery_sha256'], record['token']) path.chmod(0o600);os.link(path, self.root / 'record-alias') with self.assertRaises(SystemExit): - recovery.validate_record(self.root, subject.digest(subject.canonical(record)), 'a' * 40, + recovery.validate_record(self.root, record_sha, 'a' * 40, self.sealed.sha256, record['recovery_sha256'], record['token']) def test_foreign_owner_or_renamed_object_is_never_removed(self): From 5a665c378395c094612de5274a57f696e11eeff2 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:48:04 +0000 Subject: [PATCH 06/14] fix: verify recovery epochs and internal networks --- .../acceptance/condition-history/README.md | 17 +++- .../acceptance/condition-history/engine.py | 5 +- .../acceptance/condition-history/recover.py | 60 +++++++++--- .../acceptance/condition-history/resources.py | 34 ++++++- .../condition-history/test_audit.py | 2 +- .../condition-history/test_packet.py | 2 +- .../condition-history/test_resources.py | 47 ++++++++- .../condition-history/test_subject.py | 96 ++++++++++++++++++- 8 files changed, 235 insertions(+), 28 deletions(-) diff --git a/scripts/acceptance/condition-history/README.md b/scripts/acceptance/condition-history/README.md index d3e73fe..1d364bc 100644 --- a/scripts/acceptance/condition-history/README.md +++ b/scripts/acceptance/condition-history/README.md @@ -93,7 +93,11 @@ and cleanup. Active work ends 60 seconds before the final ceiling. The Cell's roles share one maximum 600-second deadline plus a 10-second TERM/KILL grace. `run.json` is an exclusive owner-only regular single-link record binding R, H, manifest and recovery digests, token, local daemon, root, UID/GID, isolated owner -PID/group/session epoch and all three deadlines. Its digest is captured outside +PID/group/session, Linux boot ID and kernel start ticks, and all three deadlines. +Execution requires Linux proc authority. Identity observations must agree across +two kernel-stat reads and boot reads; recovery revalidates that tuple immediately +before each TERM or KILL. A missing or changed authority refuses the signal. +Its digest is captured outside the work root in the private launch handoff. Before any pull, the owner starts a detached recovery process from captured @@ -153,6 +157,17 @@ recovery-process receipt, and confirm every matching container, network, volume, owner and recovery process group absent. It writes its own exclusive handoff. Sanitize and retain both immutable results before any acceptance or closure. +The task network has an explicit local bridge/IPv4-only IPAM profile and exact +masquerade, inter-container and loopback-binding options. Driver, scope, ingress, +attachable/configuration flags, labels, IPAM and endpoints are read back before +releasing each role. This does not claim isolation from the daemon host itself. + +The current shared-daemon execution proposal is held by Security's storage +finding on #210. A subsequent reviewed proposal must select a disposable +task-owned daemon/work store with a hard byte ceiling and externally owned +teardown. Capacity samples do not establish that ceiling. Neither the process +nor network source correction admits a pull, build or runtime from this packet. + The comparator scans every opened wait, condition outcome and timer row, binding the authored group, member key/fingerprint/occurrence, wait ID and integer timeout. Additional reopens must keep that member lineage and cannot extend its recorded diff --git a/scripts/acceptance/condition-history/engine.py b/scripts/acceptance/condition-history/engine.py index 8d1c59e..d743a21 100644 --- a/scripts/acceptance/condition-history/engine.py +++ b/scripts/acceptance/condition-history/engine.py @@ -181,10 +181,7 @@ def execute(self, args): self.cell_path('state').mkdir(parents=True) self.dir.chmod(0o777) self.state.chmod(0o777) - network = ['docker', 'network', 'create', '--internal'] - for key, value in self.controller.labels('network').items(): - network += ['--label', key + '=' + value] - command(args, network + [self.task]) + command(args, self.controller.network_arguments()) # No service roles exist during loopback replay; then exactly four # service roles plus one combined driver/Worker. No proxy/repair. replay = self.start('consumer', RUST, ['/fixture/condition-history', '--exact', diff --git a/scripts/acceptance/condition-history/recover.py b/scripts/acceptance/condition-history/recover.py index e1d3456..f3db34b 100644 --- a/scripts/acceptance/condition-history/recover.py +++ b/scripts/acceptance/condition-history/recover.py @@ -68,10 +68,43 @@ def exclusive(path, data): def process_identity(pid): - result = subprocess.run(['ps', '-p', str(pid), '-o', 'lstart='], capture_output=True, text=True, timeout=5) - if result.returncode or not result.stdout.strip(): + require(sys.platform == 'linux' and type(pid) is int and pid > 1, 'Linux proc process authority required') + boot = boot_identity() + try: + first = process_stat(pid) + second = process_stat(pid) + except FileNotFoundError: return None - return {'pid': pid, 'pgid': os.getpgid(pid), 'sid': os.getsid(pid), 'started': ' '.join(result.stdout.split())} + require(first == second and boot == boot_identity(), 'process epoch changed during observation') + return dict(first, boot_id=boot) + + +def boot_identity(): + value = pathlib.Path('/proc/sys/kernel/random/boot_id').read_text().strip() + require(re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', value), 'invalid Linux boot authority') + return value + + +def process_stat(pid): + fd = os.open('/proc/%d/stat' % pid, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as stream: + raw = stream.read(4097) + require(len(raw) <= 4096 and raw.startswith(('%d (' % pid).encode()), 'invalid proc process authority') + # comm may contain spaces or parentheses. Fields after its final ')' start + # at state (field 3); starttime is field 22 in the kernel's proc ABI. + head, separator, tail = raw.rpartition(b') ') + fields = tail.split() + require(separator and head and len(fields) >= 20, 'incomplete proc process authority') + return {'pid': pid, 'pgid': int(fields[2]), 'sid': int(fields[3]), 'start_ticks': int(fields[19])} + + +def validate_process(process, label): + require(set(process) == {'pid', 'pgid', 'sid', 'boot_id', 'start_ticks'} + and all(type(process[k]) is int for k in ('pid', 'pgid', 'sid', 'start_ticks')) + and process['pid'] > 1 and process['pid'] == process['pgid'] == process['sid'] + and process['start_ticks'] > 0 + and isinstance(process['boot_id'], str) + and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', process['boot_id']), label) def group_members(pgid): @@ -85,7 +118,7 @@ def make_record(args, manifest_sha, recovery_sha): require(owner and owner['pid'] == owner['pgid'] == owner['sid'], 'isolated owner session required') root = pathlib.Path(args.work_dir) require(root.is_absolute() and root.resolve() == root, 'canonical work root required') - return {'schema': 'durable-workflow.condition-run/v2', 'owner': PREFIX, 'token': args.run_token, + return {'schema': 'durable-workflow.condition-run/v3', 'owner': PREFIX, 'token': args.run_token, 'cells': [CELL], 'runner_commit': args.runner_commit, 'package_commit': H, 'manifest_sha256': manifest_sha, 'recovery_sha256': recovery_sha, 'started_unix': args.started_unix, 'active_unix': args.active_unix, @@ -103,7 +136,7 @@ def validate_record(root, expected_record_sha, runner, manifest_sha, recovery_sh raw = read(root / 'run.json') require(digest(raw) == expected_record_sha, 'run record digest mismatch') record = json.loads(raw) - require(set(record) == RECORD_KEYS and record['schema'] == 'durable-workflow.condition-run/v2', 'wrong run record schema') + require(set(record) == RECORD_KEYS and record['schema'] == 'durable-workflow.condition-run/v3', 'wrong run record schema') require(record['owner'] == PREFIX and record['token'] == token and re.fullmatch('[0-9a-f]{16}', token) and record['cells'] == [CELL], 'wrong recovery ownership') require(record['runner_commit'] == runner and re.fullmatch('[0-9a-f]{40}', runner) @@ -115,9 +148,7 @@ def validate_record(root, expected_record_sha, runner, manifest_sha, recovery_sh and record['cleanup_unix'] == record['started_unix'] + 1800 and record['active_unix'] == record['cleanup_unix'] - 60, 'wrong recovery deadlines') process = record['owner_process'] - require(set(process) == {'pid', 'pgid', 'sid', 'started'} and type(process['pid']) is int - and process['pid'] > 1 and process['pid'] == process['pgid'] == process['sid'] - and isinstance(process['started'], str) and process['started'], 'wrong captured owner process') + validate_process(process, 'wrong captured owner process') snapshot = root / 'snapshot' require(not snapshot.is_symlink() and stat.S_ISDIR(snapshot.lstat().st_mode), 'unsafe recovery snapshot') raw_manifest = read(snapshot / 'manifest.json', mode=0o400) @@ -236,15 +267,20 @@ def stop_owner(record): current = process_identity(expected['pid']) if current is not None: require(current == expected, 'owner PID epoch changed; refusing foreign process') - os.killpg(expected['pgid'], signal.SIGTERM) + signal_owner(expected, signal.SIGTERM) until = time.monotonic() + 5 while group_members(expected['pgid']) and time.monotonic() < until: time.sleep(0.1) if group_members(expected['pgid']): - os.killpg(expected['pgid'], signal.SIGKILL) + signal_owner(expected, signal.SIGKILL) require(not group_members(expected['pgid']), 'owner group survived or cannot be attributed') +def signal_owner(expected, signum): + require(process_identity(expected['pid']) == expected, 'owner process epoch changed before signal') + os.killpg(expected['pgid'], signum) + + def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('mode', choices=('recover', 'audit')) @@ -326,9 +362,7 @@ def main(argv=None): require(set(watcher) == {'source_sha256', 'process'} and watcher['source_sha256'] == source_sha, 'wrong captured recovery-process receipt') process = watcher['process'] - require(set(process) == {'pid', 'pgid', 'sid', 'started'} and type(process['pid']) is int - and process['pid'] > 1 and process['pid'] == process['pgid'] == process['sid'] - and isinstance(process['started'], str) and process['started'], 'wrong recovery-process identity') + validate_process(process, 'wrong recovery-process identity') require(not group_members(watcher['process']['pgid']), 'independent recovery process survivor') require(summary.get('checks_outcome') == 'pass' and summary.get('outcome') == 'pending_independent_audit' diff --git a/scripts/acceptance/condition-history/resources.py b/scripts/acceptance/condition-history/resources.py index df833ad..6cb550f 100644 --- a/scripts/acceptance/condition-history/resources.py +++ b/scripts/acceptance/condition-history/resources.py @@ -13,6 +13,11 @@ ROLE = 'dev.durable-workflow.acceptance.role' PREFIX = 'sdk-rust-condition' CELL = 'condition-history' +NETWORK_OPTIONS = {'com.docker.network.bridge.enable_ip_masquerade': 'false', + 'com.docker.network.bridge.enable_icc': 'true', + 'com.docker.network.bridge.host_binding_ipv4': '127.0.0.1'} +NETWORK_IPAM = {'Driver': 'default', 'Options': {}, + 'Config': [{'Subnet': '172.30.254.0/28', 'Gateway': '172.30.254.1'}]} PROFILES = { 'fetch': (2_000_000_000, 3 * 1024**3, 1024, True, ()), 'build': (2_000_000_000, 3 * 1024**3, 1024, True, ()), @@ -89,6 +94,32 @@ def call(self, line, **kwargs): def labels(self, role): return {RUN: self.args.run_token, OWNER: PREFIX, SUBJECT: self.subject.sha256, ROLE: role} + def network_arguments(self): + line = ['docker', 'network', 'create', '--driver', 'bridge', '--scope', 'local', '--internal', + '--ipv4=true', '--ipv6=false', '--ipam-driver', 'default', + '--subnet', NETWORK_IPAM['Config'][0]['Subnet'], '--gateway', NETWORK_IPAM['Config'][0]['Gateway']] + for key, value in NETWORK_OPTIONS.items(): + line += ['--opt', key + '=' + value] + for key, value in self.labels('network').items(): + line += ['--label', key + '=' + value] + return line + [self.task] + + def validate_network(self, network, container_id): + require(re.fullmatch('[0-9a-f]{64}', network['Id']) and network['Name'] == self.task + and network['Driver'] == 'bridge' and network['Scope'] == 'local' + and network['Internal'] is True and network['Attachable'] is False and network['Ingress'] is False + and network['EnableIPv4'] is True and network['EnableIPv6'] is False + and network['ConfigOnly'] is False and network['ConfigFrom'] == {'Network': ''}, 'wrong effective network profile') + require(network['Labels'] == self.labels('network') and network['Options'] == NETWORK_OPTIONS, + 'wrong internal network authority/options') + ipam = dict(network['IPAM'], Options=network['IPAM'].get('Options') or {}) + require(ipam == NETWORK_IPAM, 'wrong effective network IPAM') + allowed = {r['container_id'] for r in self.receipts if r['network'] == self.task} | {container_id} + require(set(network['Containers']) <= allowed and container_id in network['Containers'], 'foreign or missing network endpoint') + return {'id': network['Id'], 'driver': 'bridge', 'scope': 'local', 'internal': True, + 'attachable': False, 'ingress': False, 'ipv4': True, 'ipv6': False, + 'options': NETWORK_OPTIONS, 'ipam': NETWORK_IPAM} + def image(self, reference): if reference not in self.images: data = json.loads(self.call(['docker', 'image', 'inspect', reference]).stdout)[0] @@ -180,7 +211,7 @@ def validate(self, spec, data): 'wrong effective network/device sharing') if spec['network'] == self.task: network = json.loads(self.call(['docker', 'network', 'inspect', self.task]).stdout)[0] - require(network['Internal'] is True and all(network['Labels'].get(k) == v for k, v in self.labels('network').items()), 'wrong internal network authority') + network_profile = self.validate_network(network, data['Id']) aliases = set(data['NetworkSettings']['Networks'].get(self.task, {}).get('Aliases') or []) require(not spec['alias'] or spec['alias'] in aliases, 'missing effective network alias') require(set(data['NetworkSettings']['Networks']) == {spec['network']}, 'unexpected attached network') @@ -192,6 +223,7 @@ def validate(self, spec, data): require({m['Destination'] for m in data['Mounts'] if m['Type'] == 'tmpfs'} == set(spec['tmpfs']) and len(data['Mounts']) == len(spec['tmpfs']) + len(spec['binds']), 'missing, duplicate or unexpected mount') return {'role': spec['role'], 'container_id': data['Id'], 'image': spec['image'], + 'network_profile': network_profile if spec['network'] == self.task else None, 'image_content_sha256': self.image(spec['image'])['sha256'], 'user': spec['user'], 'memory': spec['memory'], 'memory_swap': spec['memory'], 'nano_cpus': spec['cpus'], 'pids': spec['pids'], 'readonly': spec['readonly'], 'cap_drop': ['ALL'], 'cap_add': spec['cap_add'], diff --git a/scripts/acceptance/condition-history/test_audit.py b/scripts/acceptance/condition-history/test_audit.py index e981639..d325a48 100644 --- a/scripts/acceptance/condition-history/test_audit.py +++ b/scripts/acceptance/condition-history/test_audit.py @@ -25,7 +25,7 @@ def setUp(self): self.record_sha = subject.digest(subject.canonical(self.authority)) self.program = self.sealed.captured[subject.PREFIX + 'recover.py'].decode() self.watcher = {'source_sha256': self.authority['recovery_sha256'], - 'process': {'pid': 23456, 'pgid': 23456, 'sid': 23456, 'started': 'synthetic watcher'}} + 'process': {'pid': 23456, 'pgid': 23456, 'sid': 23456, 'boot_id': '11111111-1111-1111-1111-111111111111', 'start_ticks': 234567}} watcher_sha = subject.exclusive(self.root / 'recovery-process.json', subject.canonical(self.watcher)) self.summary = {'checks_outcome': 'pass', 'outcome': 'pending_independent_audit', 'record_sha256': self.record_sha, 'runner_commit': self.authority['runner_commit'], diff --git a/scripts/acceptance/condition-history/test_packet.py b/scripts/acceptance/condition-history/test_packet.py index 4ccbeaa..5954b54 100644 --- a/scripts/acceptance/condition-history/test_packet.py +++ b/scripts/acceptance/condition-history/test_packet.py @@ -54,7 +54,7 @@ def test_no_optimizable_runtime_assert(self): def recovery_start(self, changed=None, process_missing=False, exited=False): pathlib.Path(self.args.work_dir).mkdir() subject.RECORD_SHA = 'c' * 64 - process = {'pid': 23456, 'pgid': 23456, 'sid': 23456, 'started': 'synthetic watcher'} + process = {'pid': 23456, 'pgid': 23456, 'sid': 23456, 'boot_id': '11111111-1111-1111-1111-111111111111', 'start_ticks': 234567} recovery_sha = subject.CURRENT.value['files'][subject.PREFIX + 'recover.py']['sha256'] ready = {'record_sha256': subject.RECORD_SHA, 'manifest_sha256': subject.CURRENT.sha256, 'recovery_sha256': recovery_sha, 'process': process} diff --git a/scripts/acceptance/condition-history/test_resources.py b/scripts/acceptance/condition-history/test_resources.py index 688029c..66d88fd 100644 --- a/scripts/acceptance/condition-history/test_resources.py +++ b/scripts/acceptance/condition-history/test_resources.py @@ -55,9 +55,15 @@ def frame(self, role): 'Mounts': [{'Type': 'bind', 'Source': s, 'Destination': d, 'RW': rw} for s, d, rw in role['binds']] + [{'Type':'tmpfs', 'Destination':d} for d in role['tmpfs']], 'NetworkSettings': {'Networks': {role['network']: {'Aliases': [role['alias']] if role['alias'] else []}}}} + def network_frame(self): + return {'Id': 'e' * 64, 'Name': self.controller.task, 'Driver': 'bridge', 'Scope': 'local', + 'Internal': True, 'Attachable': False, 'Ingress': False, 'EnableIPv4': True, 'EnableIPv6': False, + 'ConfigOnly': False, 'ConfigFrom': {'Network': ''}, 'Labels': self.controller.labels('network'), + 'IPAM': copy.deepcopy(resources.NETWORK_IPAM), 'Options': dict(resources.NETWORK_OPTIONS), + 'Containers': {'d' * 64: {}}} + def network(self): - return types.SimpleNamespace(stdout=json.dumps([{'Internal': True, - 'Labels': self.controller.labels('network')}])) + return types.SimpleNamespace(stdout=json.dumps([self.network_frame()])) def test_actual_held_build_has_no_swap_cap(self): repo=HERE.parents[2] @@ -113,8 +119,8 @@ def test_each_bad_effective_field_refuses_including_offline_build(self): def test_internal_network_and_mount_authority_are_effective_checks(self): role = self.role() frame = self.frame(role) - with mock.patch.object(self.controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([ - {'Internal': False, 'Labels': self.controller.labels('network')}]))), self.assertRaises(SystemExit): + network = self.network_frame();network['Internal'] = False + with mock.patch.object(self.controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([network]))), self.assertRaises(SystemExit): self.controller.validate(role, frame) for mutate in (lambda f: f['Mounts'].append({'Type': 'volume', 'Destination': '/extra'}), lambda f: f['Mounts'].pop(), @@ -187,5 +193,38 @@ def cli(args, line, **kwargs): self.assertFalse(pathlib.Path(self.args.work_dir).exists()) self.assertFalse(any(x in line for line in ledger for x in ('create', 'pull', 'start', 'rm'))) + def test_complete_network_profile_and_ipam_are_read_back(self): + role = self.role();frame = self.frame(role) + cases = [('Driver', 'overlay'), ('Scope', 'swarm'), ('Internal', False), ('Attachable', True), + ('Ingress', True), ('EnableIPv4', False), ('EnableIPv6', True), ('ConfigOnly', True), + ('ConfigFrom', {'Network': 'foreign'}), ('Options', {}), ('Labels', {}), + ('IPAM', dict(resources.NETWORK_IPAM, Driver='foreign')), + ('IPAM', dict(resources.NETWORK_IPAM, Config=[{'Subnet': '10.0.0.0/8'}])), + ('IPAM', dict(resources.NETWORK_IPAM, Options={'foreign': 'true'})), + ('Containers', {'d' * 64: {}, 'f' * 64: {}})] + for key, value in cases: + network = self.network_frame();network[key] = value + with self.subTest(key=key, value=value), mock.patch.object(self.controller, 'call', + return_value=types.SimpleNamespace(stdout=json.dumps([network]))), self.assertRaises(SystemExit): + self.controller.validate(role, frame) + args = self.controller.network_arguments() + self.assertIn('--ipv4=true', args);self.assertIn('--ipv6=false', args) + self.assertEqual(args[args.index('--subnet') + 1], '172.30.254.0/28') + + def test_held_network_check_accepts_foreign_driver_scope_and_ipam(self): + source = subprocess.check_output(['git', '--no-replace-objects', '-C', str(HERE.parents[2]), 'show', + 'e309116cb0dd79e415ae6a25b809515354cea026:scripts/acceptance/condition-history/resources.py']) + old = types.ModuleType('held_network_resource_controls') + exec(compile(source, 'held-e309-resources', 'exec'), old.__dict__) + controller = old.Docker(self.args, Frozen(), {}) + controller.images = dict(self.controller.images) + role = self.role();frame = self.frame(role) + network = self.network_frame();network.update(Driver='overlay', Scope='swarm', IPAM={'Driver': 'foreign'}) + with mock.patch.object(controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([network]))): + self.assertEqual(controller.validate(role, frame)['container_id'], frame['Id']) + with mock.patch.object(self.controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([network]))), \ + self.assertRaisesRegex(SystemExit, 'network profile'): + self.controller.validate(role, frame) + if __name__ == '__main__':unittest.main() diff --git a/scripts/acceptance/condition-history/test_subject.py b/scripts/acceptance/condition-history/test_subject.py index c6a5f2e..1bf65c4 100644 --- a/scripts/acceptance/condition-history/test_subject.py +++ b/scripts/acceptance/condition-history/test_subject.py @@ -172,14 +172,14 @@ def test_exclusive_summary_handoff_rejects_existing_file_and_redirect(self): def record(self): now = int(time.time()) - return {'schema': 'durable-workflow.condition-run/v2', 'owner': recovery.PREFIX, + return {'schema': 'durable-workflow.condition-run/v3', 'owner': recovery.PREFIX, 'token': '0123456789abcdef', 'cells': [recovery.CELL], 'runner_commit': 'a' * 40, 'package_commit': subject.H, 'manifest_sha256': self.sealed.sha256, 'recovery_sha256': self.sealed.value['files'][subject.PREFIX + 'recover.py']['sha256'], 'started_unix': now, 'active_unix': now + 1740, 'cleanup_unix': now + 1800, 'work_dir': str(self.root), 'docker_endpoint': 'unix:///synthetic.sock', 'uid': os.getuid(), 'gid': os.getgid(), - 'owner_process': {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'started': 'synthetic epoch'}} + 'owner_process': {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'boot_id': '11111111-1111-1111-1111-111111111111', 'start_ticks': 123456}} def validate(self, record, expected_sha=None): path = self.root / 'run.json' @@ -234,11 +234,101 @@ def test_foreign_owner_or_renamed_object_is_never_removed(self): self.assertEqual(cli.call_count, 1) def test_changed_pid_epoch_never_kills_a_foreign_group(self): - record = self.record();changed = dict(record['owner_process'], started='other epoch') + record = self.record();changed = dict(record['owner_process'], start_ticks=654321) with mock.patch.object(recovery, 'process_identity', return_value=changed), \ mock.patch.object(recovery.os, 'killpg') as kill, self.assertRaises(SystemExit): recovery.stop_owner(record) kill.assert_not_called() + def test_held_whole_second_identity_accepts_a_recycled_session(self): + source = subprocess.check_output(['git', '--no-replace-objects', '-C', str(REPO), 'show', + 'e309116cb0dd79e415ae6a25b809515354cea026:scripts/acceptance/condition-history/recover.py']) + old = types.ModuleType('held_second_resolution_recovery') + exec(compile(source, 'held-e309-recovery', 'exec'), old.__dict__) + # Both incarnations have identical numeric IDs and ps whole-second text. + old.subprocess = types.SimpleNamespace(run=lambda *a, **k: types.SimpleNamespace( + returncode=0, stdout='Thu Nov 5 01:00:00 2026')) + old.os = types.SimpleNamespace(getpgid=lambda pid: pid, getsid=lambda pid: pid, killpg=mock.Mock()) + old.group_members = lambda pgid: [] + legacy = old.process_identity(12345) + old.stop_owner({'owner_process': legacy}) + old.os.killpg.assert_called_once_with(12345, old.signal.SIGTERM) + record = self.record() + for delta in ({'start_ticks': record['owner_process']['start_ticks'] + 1}, + {'boot_id': '22222222-2222-2222-2222-222222222222'}): + recycled = dict(record['owner_process'], **delta) + with self.subTest(delta=delta), mock.patch.object(recovery, 'process_identity', return_value=recycled), \ + mock.patch.object(recovery.os, 'killpg') as kill, self.assertRaisesRegex(SystemExit, 'epoch changed'): + recovery.stop_owner(record) + kill.assert_not_called() + + def test_proc_stat_reads_kernel_fields_after_a_parenthesized_comm(self): + fields = ['R', '1', '12345', '12345'] + ['0'] * 15 + ['123456'] + path = self.root / 'proc-stat' + path.write_bytes(('12345 (comm with ) spaces (and)) ' + ' '.join(fields)).encode()) + original_open = os.open + with mock.patch.object(recovery.os, 'open', side_effect=lambda path_, flags: original_open(path, flags)): + result = recovery.process_stat(12345) + self.assertEqual(result, {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'start_ticks': 123456}) + + def test_inconsistent_or_unavailable_kernel_authority_refuses(self): + first = {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'start_ticks': 123456} + boot = self.record()['owner_process']['boot_id'] + with mock.patch.object(recovery, 'boot_identity', return_value=boot), \ + mock.patch.object(recovery, 'process_stat', side_effect=[first, dict(first, start_ticks=123457)]), \ + self.assertRaisesRegex(SystemExit, 'during observation'): + recovery.process_identity(12345) + with mock.patch.object(recovery, 'boot_identity', side_effect=[boot, '22222222-2222-2222-2222-222222222222']), \ + mock.patch.object(recovery, 'process_stat', return_value=first), \ + self.assertRaisesRegex(SystemExit, 'during observation'): + recovery.process_identity(12345) + for error in (PermissionError('denied'), OSError('proc unavailable')): + with self.subTest(error=type(error).__name__), mock.patch.object(recovery, 'boot_identity', return_value=boot), \ + mock.patch.object(recovery, 'process_stat', side_effect=error), self.assertRaises(OSError): + recovery.process_identity(12345) + with mock.patch.object(recovery, 'boot_identity', return_value=boot), \ + mock.patch.object(recovery, 'process_stat', side_effect=FileNotFoundError()): + self.assertIsNone(recovery.process_identity(12345)) + + def test_epoch_revalidated_immediately_before_term_and_kill(self): + record = self.record();expected = record['owner_process'] + recycled = dict(expected, start_ticks=expected['start_ticks'] + 1) + with mock.patch.object(recovery, 'process_identity', side_effect=[expected, recycled]), \ + mock.patch.object(recovery.os, 'killpg') as kill, self.assertRaisesRegex(SystemExit, 'before signal'): + recovery.stop_owner(record) + kill.assert_not_called() + with mock.patch.object(recovery, 'process_identity', side_effect=[expected, expected, recycled]), \ + mock.patch.object(recovery, 'group_members', return_value=[expected['pid']]), \ + mock.patch.object(recovery.time, 'monotonic', side_effect=[0, 10]), \ + mock.patch.object(recovery.os, 'killpg') as kill, self.assertRaisesRegex(SystemExit, 'before signal'): + recovery.stop_owner(record) + kill.assert_called_once_with(expected['pgid'], recovery.signal.SIGTERM) + + def test_exact_epoch_can_receive_both_signals_with_fresh_authority(self): + record = self.record();expected = record['owner_process'] + with mock.patch.object(recovery, 'process_identity', return_value=expected) as observe, \ + mock.patch.object(recovery, 'group_members', side_effect=[[expected['pid']], [expected['pid']], []]), \ + mock.patch.object(recovery.time, 'monotonic', side_effect=[0, 10]), \ + mock.patch.object(recovery.os, 'killpg') as kill: + recovery.stop_owner(record) + self.assertEqual(observe.call_count, 3) + self.assertEqual(kill.call_args_list, [mock.call(expected['pgid'], recovery.signal.SIGTERM), + mock.call(expected['pgid'], recovery.signal.SIGKILL)]) + + def test_legacy_process_record_cannot_authorize_a_signal(self): + record = self.record() + record['owner_process'] = {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'started': 'whole second'} + with self.assertRaisesRegex(SystemExit, 'captured owner process'): + self.validate(record) + + @unittest.skipUnless(sys.platform == 'linux', 'Linux proc authority is the execution contract') + def test_readonly_current_kernel_identity(self): + identity = recovery.process_identity(os.getpid()) + self.assertEqual(identity['pid'], os.getpid()) + self.assertEqual(identity['pgid'], os.getpgid(os.getpid())) + self.assertEqual(identity['sid'], os.getsid(os.getpid())) + self.assertGreater(identity['start_ticks'], 0) + self.assertEqual(identity['boot_id'], recovery.boot_identity()) + if __name__ == '__main__':unittest.main() From fd00f1e41fd81fb049d0e34406cc2dec11a92c51 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:20:15 +0000 Subject: [PATCH 07/14] fix: refuse unbounded guest work and daemon storage --- .../acceptance/condition-history/README.md | 15 +++- .../acceptance/condition-history/engine.py | 9 ++- .../acceptance/condition-history/recover.py | 78 +++++++++++++++++-- .../acceptance/condition-history/resources.py | 2 + .../acceptance/condition-history/subject.py | 7 +- .../condition-history/test_audit.py | 2 + .../condition-history/test_packet.py | 2 +- .../condition-history/test_subject.py | 72 ++++++++++++++++- 8 files changed, 176 insertions(+), 11 deletions(-) diff --git a/scripts/acceptance/condition-history/README.md b/scripts/acceptance/condition-history/README.md index 1d364bc..38141a6 100644 --- a/scripts/acceptance/condition-history/README.md +++ b/scripts/acceptance/condition-history/README.md @@ -130,6 +130,7 @@ raise SystemExit(subprocess.run([sys.executable, "-I", "-S", "-c", entry, ' CHECKOUT R --manifest-sha256 M \ --package-commit 4b2440a5618ccc1c98960ea2e1e62127042c3590 \ --work-dir W --run-token T --docker-endpoint D \ + --guest-boot-id B \ --started-unix S --active-unix ACTIVE --cleanup-unix CLEANUP ``` @@ -155,7 +156,10 @@ remain `pending_independent_audit`. Only after owner exit can a separately invok captured audit bind that summary digest, verify the original record/snapshot and recovery-process receipt, and confirm every matching container, network, volume, owner and recovery process group absent. It writes its own exclusive handoff. -Sanitize and retain both immutable results before any acceptance or closure. +The guest audit returns `pending_external_guest_teardown`, not acceptance. +Sanitize and retain both immutable results before the external guardian removes +the guest and its enclosing store. Independent host absence/custody evidence +remains required before any acceptance or closure. The task network has an explicit local bridge/IPv4-only IPAM profile and exact masquerade, inter-container and loopback-binding options. Driver, scope, ingress, @@ -168,6 +172,15 @@ task-owned daemon/work store with a hard byte ceiling and externally owned teardown. Capacity samples do not establish that ceiling. Neither the process nor network source correction admits a pull, build or runtime from this packet. +The guest entry now additionally binds the externally observed boot ID B, a +block-backed ext4 root of at most 20 GiB, and the same device for its fresh work +parent and Docker data root. It refuses host/network filesystem shares, a +nonempty daemon or build-cache baseline, and a later device/geometry drift. +The run record binds that storage identity. Image/cache inventories remain in +external-guardian custody until whole-guest/store teardown; the guest never +guesses which layers to delete from a shared daemon. The concrete host guardian +and guest proposal are still being prepared and are not admitted for execution. + The comparator scans every opened wait, condition outcome and timer row, binding the authored group, member key/fingerprint/occurrence, wait ID and integer timeout. Additional reopens must keep that member lineage and cannot extend its recorded diff --git a/scripts/acceptance/condition-history/engine.py b/scripts/acceptance/condition-history/engine.py index d743a21..3eea9b8 100644 --- a/scripts/acceptance/condition-history/engine.py +++ b/scripts/acceptance/condition-history/engine.py @@ -259,7 +259,7 @@ def execute(args): watchdog, recovery_process_sha = start_recovery(args, record) result = {'outcome': 'fail', 'checks_outcome': 'fail', 'record_sha256': subject.RECORD_SHA, 'runner_commit': args.runner_commit, 'package_commit': H, 'manifest_sha256': subject.CURRENT.sha256, - 'recovery_process_sha256': recovery_process_sha} + 'recovery_process_sha256': recovery_process_sha, 'storage': record['storage']} try: registry = registry_identity() images = [RUST, SERVER, MYSQL, REDIS] @@ -310,6 +310,13 @@ def execute(args): cleanup = recovery.cleanup(record) result['cleanup'] = cleanup result['cleanup_errors'] = owner.cleanup_errors + try: + result['storage_custody'] = {'disposition': 'pending_external_guest_teardown', + 'image_ids': recovery.docker(record, ['image', 'ls', '-q', '--no-trunc']).stdout.split(), + 'image_cache_inventory': [json.loads(line) for line in recovery.docker(record, + ['system', 'df', '--format', '{{json .}}']).stdout.splitlines() if line]} + except BaseException: + result.update(outcome='fail', checks_outcome='fail', storage_custody={'disposition': 'unverified'}) if not cleanup['pass'] or owner.cleanup_errors: result.update(outcome='fail', checks_outcome='fail') subject.CURRENT.verify() diff --git a/scripts/acceptance/condition-history/recover.py b/scripts/acceptance/condition-history/recover.py index f3db34b..baf07a0 100644 --- a/scripts/acceptance/condition-history/recover.py +++ b/scripts/acceptance/condition-history/recover.py @@ -20,7 +20,9 @@ ROLE = 'dev.durable-workflow.acceptance.role' RECORD_KEYS = {'schema', 'owner', 'token', 'cells', 'runner_commit', 'package_commit', 'manifest_sha256', 'recovery_sha256', 'started_unix', 'active_unix', 'cleanup_unix', 'work_dir', - 'docker_endpoint', 'uid', 'gid', 'owner_process'} + 'docker_endpoint', 'uid', 'gid', 'owner_process', 'storage'} +STORAGE_KEYS = {'schema', 'guest_boot_id', 'device', 'block_bytes', 'filesystem_bytes', 'docker_root', 'daemon_id'} +GUEST_DISK_BYTES = 20 * 1024**3 ROLES = ('fetch', 'build', 'mysql', 'redis', 'bootstrap', 'runtime', 'queue', 'consumer') PUBLIC_CONFIG = {'auths': {'https://index.docker.io/v1/': {}, 'ghcr.io': {}}} @@ -107,6 +109,66 @@ def validate_process(process, label): and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', process['boot_id']), label) +def filesystem_storage(work_dir, docker_root, expected_boot): + require(sys.platform == 'linux' and boot_identity() == expected_boot, 'wrong disposable guest boot identity') + root = pathlib.Path('/') + device = root.stat().st_dev + require(os.major(device) != 0, 'block-backed disposable guest root required') + for name in (pathlib.Path(work_dir).parent, pathlib.Path(docker_root)): + require(name.is_absolute() and name.resolve(strict=True) == name and name.stat().st_dev == device, + 'work/daemon storage escapes guest root device') + block_path = pathlib.Path('/sys/dev/block/%d:%d/size' % (os.major(device), os.minor(device))) + block_bytes = int(block_path.read_text().strip()) * 512 + fs = os.statvfs(root) + filesystem_bytes = fs.f_blocks * fs.f_frsize + require(0 < filesystem_bytes <= block_bytes <= GUEST_DISK_BYTES, 'guest storage exceeds hard block ceiling') + mounts = pathlib.Path('/proc/self/mountinfo').read_text().splitlines() + root_rows = [] + for line in mounts: + before, separator, after = line.partition(' - ') + require(separator, 'invalid guest mount authority') + fields, tail = before.split(), after.split() + require(len(fields) >= 6 and len(tail) >= 3, 'invalid guest mount authority') + require(tail[0] not in ('9p', 'virtiofs', 'nfs', 'nfs4', 'cifs', 'fuse.sshfs'), 'host/network filesystem share refused') + if fields[4] == '/': + root_rows.append((fields, tail)) + require(len(root_rows) == 1 and root_rows[0][1][0] == 'ext4' + and root_rows[0][0][2] == '%d:%d' % (os.major(device), os.minor(device)), 'wrong guest root filesystem') + return {'guest_boot_id': expected_boot, 'device': '%d:%d' % (os.major(device), os.minor(device)), + 'block_bytes': block_bytes, 'filesystem_bytes': filesystem_bytes, 'docker_root': docker_root} + + +def validate_storage(value, work_dir, observe=True): + require(set(value) == STORAGE_KEYS and value['schema'] == 'durable-workflow.condition-guest-storage/v1' + and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', value['guest_boot_id']) + and re.fullmatch('[0-9]+:[0-9]+', value['device']) + and type(value['block_bytes']) is int and type(value['filesystem_bytes']) is int + and 0 < value['filesystem_bytes'] <= value['block_bytes'] <= GUEST_DISK_BYTES + and isinstance(value['docker_root'], str) and value['docker_root'].startswith('/') + and isinstance(value['daemon_id'], str) and re.fullmatch('[A-Za-z0-9:-]{12,128}', value['daemon_id']), + 'wrong bounded guest storage authority') + if observe: + current = filesystem_storage(work_dir, value['docker_root'], value['guest_boot_id']) + require(all(value[k] == v for k, v in current.items()), 'guest storage authority changed') + + +def storage_preflight(args, command): + info = json.loads(command(['docker', 'info', '--format', '{{json .}}']).stdout) + frame = filesystem_storage(args.work_dir, info['DockerRootDir'], args.guest_boot_id) + value = dict(frame, schema='durable-workflow.condition-guest-storage/v1', daemon_id=info['ID']) + validate_storage(value, args.work_dir) + # The public base image contains tools only. Nothing in this daemon may be + # inherited from a previous SDK experiment or another owner. + for line in (['docker', 'ps', '-aq'], ['docker', 'image', 'ls', '-q', '--no-trunc'], ['docker', 'volume', 'ls', '-q']): + require(not command(line).stdout.split(), 'disposable guest daemon is not empty') + rows = [json.loads(line) for line in command(['docker', 'system', 'df', '--format', '{{json .}}']).stdout.splitlines() if line] + require(len(rows) == 4 and {row['Type'] for row in rows} == {'Images', 'Containers', 'Local Volumes', 'Build Cache'} + and all((type(row['TotalCount']) is int and row['TotalCount'] == 0) + or (type(row['TotalCount']) is str and row['TotalCount'] == '0') for row in rows), + 'guest image/cache baseline is not empty') + return value + + def group_members(pgid): result = subprocess.run(['ps', '-axo', 'pid=,pgid='], capture_output=True, text=True, check=True, timeout=5) return [int(fields[0]) for line in result.stdout.splitlines() @@ -118,13 +180,13 @@ def make_record(args, manifest_sha, recovery_sha): require(owner and owner['pid'] == owner['pgid'] == owner['sid'], 'isolated owner session required') root = pathlib.Path(args.work_dir) require(root.is_absolute() and root.resolve() == root, 'canonical work root required') - return {'schema': 'durable-workflow.condition-run/v3', 'owner': PREFIX, 'token': args.run_token, + return {'schema': 'durable-workflow.condition-run/v4', 'owner': PREFIX, 'token': args.run_token, 'cells': [CELL], 'runner_commit': args.runner_commit, 'package_commit': H, 'manifest_sha256': manifest_sha, 'recovery_sha256': recovery_sha, 'started_unix': args.started_unix, 'active_unix': args.active_unix, 'cleanup_unix': args.cleanup_unix, 'work_dir': str(root), 'docker_endpoint': args.docker_endpoint, 'uid': os.getuid(), 'gid': os.getgid(), - 'owner_process': owner} + 'owner_process': owner, 'storage': args.guest_storage} def validate_record(root, expected_record_sha, runner, manifest_sha, recovery_sha, token): @@ -136,7 +198,7 @@ def validate_record(root, expected_record_sha, runner, manifest_sha, recovery_sh raw = read(root / 'run.json') require(digest(raw) == expected_record_sha, 'run record digest mismatch') record = json.loads(raw) - require(set(record) == RECORD_KEYS and record['schema'] == 'durable-workflow.condition-run/v3', 'wrong run record schema') + require(set(record) == RECORD_KEYS and record['schema'] == 'durable-workflow.condition-run/v4', 'wrong run record schema') require(record['owner'] == PREFIX and record['token'] == token and re.fullmatch('[0-9a-f]{16}', token) and record['cells'] == [CELL], 'wrong recovery ownership') require(record['runner_commit'] == runner and re.fullmatch('[0-9a-f]{40}', runner) @@ -149,6 +211,8 @@ def validate_record(root, expected_record_sha, runner, manifest_sha, recovery_sh and record['active_unix'] == record['cleanup_unix'] - 60, 'wrong recovery deadlines') process = record['owner_process'] validate_process(process, 'wrong captured owner process') + validate_storage(record['storage'], root, observe=False) + require(process['boot_id'] == record['storage']['guest_boot_id'], 'owner/storage guest identity mismatch') snapshot = root / 'snapshot' require(not snapshot.is_symlink() and stat.S_ISDIR(snapshot.lstat().st_mode), 'unsafe recovery snapshot') raw_manifest = read(snapshot / 'manifest.json', mode=0o400) @@ -182,6 +246,7 @@ def unreadable(error): def docker(record, command, check=True): + validate_storage(record['storage'], record['work_dir']) root = pathlib.Path(record['work_dir']) require(json.loads(read(root / 'recovery-docker/config.json')) == PUBLIC_CONFIG, 'anonymous Docker configuration changed') env = {'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'HOME': str(root / 'public-home'), @@ -369,9 +434,10 @@ def main(argv=None): and summary.get('cleanup') == {'pass': True, 'errors': [], 'remaining': {'container': [], 'network': [], 'volume': []}} and summary.get('cleanup_errors') == [], 'owner checks or cleanup did not pass') - result = {'outcome': 'pass', 'record_sha256': args.record_sha256, + result = {'outcome': 'pending_external_guest_teardown', 'record_sha256': args.record_sha256, 'manifest_sha256': args.manifest_sha256, 'summary_sha256': args.summary_sha256, - 'remaining': remaining, 'owner_process_absent': True, 'recovery_process_absent': True} + 'remaining': remaining, 'owner_process_absent': True, 'recovery_process_absent': True, + 'storage': record['storage']} exclusive(pathlib.Path(args.work_dir) / 'independent-audit.json', canonical(result)) print(json.dumps(result)) diff --git a/scripts/acceptance/condition-history/resources.py b/scripts/acceptance/condition-history/resources.py index 6cb550f..8c15164 100644 --- a/scripts/acceptance/condition-history/resources.py +++ b/scripts/acceptance/condition-history/resources.py @@ -48,6 +48,8 @@ def environment(args): def call(args, line, timeout=120, check=True, **kwargs): + if hasattr(args, 'storage_guard'): + args.storage_guard() remaining = int(args.active_unix - time.time()) require(remaining > 0, 'active deadline exhausted') result = subprocess.run(line, capture_output=True, timeout=min(timeout, remaining), diff --git a/scripts/acceptance/condition-history/subject.py b/scripts/acceptance/condition-history/subject.py index 2190774..e45cc66 100644 --- a/scripts/acceptance/condition-history/subject.py +++ b/scripts/acceptance/condition-history/subject.py @@ -168,6 +168,7 @@ def arguments(argv=None): parser.add_argument('--work-dir') parser.add_argument('--run-token') parser.add_argument('--docker-endpoint') + parser.add_argument('--guest-boot-id') for key in ('started-unix', 'active-unix', 'cleanup-unix'): parser.add_argument('--' + key, type=int) parser.add_argument('--record-sha256') @@ -185,6 +186,8 @@ def validate(args): require(args.work_dir and not os.path.lexists(args.work_dir), 'work directory is not fresh') require(args.run_token and re.fullmatch('[0-9a-f]{16}', args.run_token), 'invalid run token') require(args.docker_endpoint and args.docker_endpoint.startswith('unix:///'), 'local daemon endpoint required') + require(args.guest_boot_id and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', args.guest_boot_id), + 'exact disposable guest boot identity required') require(type(args.started_unix) is int and type(args.active_unix) is int and type(args.cleanup_unix) is int and args.cleanup_unix == args.started_unix + 1800 and args.active_unix == args.cleanup_unix - 60, 'wrong absolute deadline envelope') @@ -221,10 +224,12 @@ def bootstrap(): module = types.ModuleType('sealed_resource_preflight') module.__file__ = 'git:' + args.runner_commit + ':' + PREFIX + 'resources.py' exec(compile(captured[PREFIX + 'resources.py'], module.__file__, 'exec'), module.__dict__) - module.preflight_collisions(args) recovery = types.ModuleType('captured_recovery_preflight') recovery.__file__ = 'git:' + args.runner_commit + ':' + PREFIX + 'recover.py' exec(compile(captured[PREFIX + 'recover.py'], recovery.__file__, 'exec'), recovery.__dict__) + args.guest_storage = recovery.storage_preflight(args, lambda line: module.call(args, line)) + args.storage_guard = lambda: recovery.validate_storage(args.guest_storage, args.work_dir) + module.preflight_collisions(args) RECORD = recovery.make_record(args, actual, value['files'][PREFIX + 'recover.py']['sha256']) RECORD_SHA = digest(canonical(RECORD)) print(json.dumps({'private_launch': RECORD, 'record_sha256': RECORD_SHA}), flush=True) diff --git a/scripts/acceptance/condition-history/test_audit.py b/scripts/acceptance/condition-history/test_audit.py index d325a48..b7cfd7f 100644 --- a/scripts/acceptance/condition-history/test_audit.py +++ b/scripts/acceptance/condition-history/test_audit.py @@ -56,6 +56,8 @@ def test_independent_audit_binds_exact_summary_and_both_processes(self): self.assertEqual(result['summary_sha256'], self.summary_sha) self.assertEqual(result['remaining'], {'container': [], 'network': [], 'volume': []}) self.assertTrue(result['owner_process_absent'] and result['recovery_process_absent']) + self.assertEqual(result['outcome'], 'pending_external_guest_teardown') + self.assertEqual(result['storage'], self.authority['storage']) with self.assertRaises(FileExistsError):self.invoke() def test_changed_summary_never_gains_authority(self): diff --git a/scripts/acceptance/condition-history/test_packet.py b/scripts/acceptance/condition-history/test_packet.py index 5954b54..21f9f7d 100644 --- a/scripts/acceptance/condition-history/test_packet.py +++ b/scripts/acceptance/condition-history/test_packet.py @@ -20,7 +20,7 @@ def setUp(self): now = int(time.time()) self.args = types.SimpleNamespace(mode='run', checkout=str(REPO), work_dir=str(self.root / 'fresh'), package_commit=subject.H, runner_commit='a' * 40, run_token='0123456789abcdef', - docker_endpoint='unix:///synthetic.sock', started_unix=now, active_unix=now+1740, cleanup_unix=now+1800) + docker_endpoint='unix:///synthetic.sock', guest_boot_id='11111111-1111-1111-1111-111111111111', started_unix=now, active_unix=now+1740, cleanup_unix=now+1800) subject.ARGS = self.args subject.CURRENT = fixture_subject(self.root) self.packet = subject.CURRENT.module('engine') diff --git a/scripts/acceptance/condition-history/test_subject.py b/scripts/acceptance/condition-history/test_subject.py index 1bf65c4..55a6dab 100644 --- a/scripts/acceptance/condition-history/test_subject.py +++ b/scripts/acceptance/condition-history/test_subject.py @@ -172,13 +172,14 @@ def test_exclusive_summary_handoff_rejects_existing_file_and_redirect(self): def record(self): now = int(time.time()) - return {'schema': 'durable-workflow.condition-run/v3', 'owner': recovery.PREFIX, + return {'schema': 'durable-workflow.condition-run/v4', 'owner': recovery.PREFIX, 'token': '0123456789abcdef', 'cells': [recovery.CELL], 'runner_commit': 'a' * 40, 'package_commit': subject.H, 'manifest_sha256': self.sealed.sha256, 'recovery_sha256': self.sealed.value['files'][subject.PREFIX + 'recover.py']['sha256'], 'started_unix': now, 'active_unix': now + 1740, 'cleanup_unix': now + 1800, 'work_dir': str(self.root), 'docker_endpoint': 'unix:///synthetic.sock', 'uid': os.getuid(), 'gid': os.getgid(), + 'storage': {'schema': 'durable-workflow.condition-guest-storage/v1', 'guest_boot_id': '11111111-1111-1111-1111-111111111111', 'device': '8:1', 'block_bytes': 20 * 1024**3, 'filesystem_bytes': 19 * 1024**3, 'docker_root': '/synthetic/docker', 'daemon_id': 'synthetic-daemon-id'}, 'owner_process': {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'boot_id': '11111111-1111-1111-1111-111111111111', 'start_ticks': 123456}} def validate(self, record, expected_sha=None): @@ -330,5 +331,74 @@ def test_readonly_current_kernel_identity(self): self.assertGreater(identity['start_ticks'], 0) self.assertEqual(identity['boot_id'], recovery.boot_identity()) + def storage_frame(self): + return {k: v for k, v in self.record()['storage'].items() if k not in ('schema', 'daemon_id')} + + def observe_storage(self, *, block_bytes=None, foreign_device=False, mount_text=None, boot=None): + device = os.makedev(8, 1) + capacity = block_bytes or recovery.GUEST_DISK_BYTES + def read_text(path, *args, **kwargs): + if str(path).startswith('/sys/dev/block/'): + return str(capacity // 512) + return mount_text or '42 1 8:1 / / rw,nodev,nosuid - ext4 /dev/sda1 rw\n' + def file_stat(path, *args, **kwargs): + return types.SimpleNamespace(st_dev=os.makedev(8, 2) if foreign_device and str(path) == '/synthetic/docker' else device) + with mock.patch.object(recovery, 'boot_identity', return_value=boot or self.record()['storage']['guest_boot_id']), \ + mock.patch.object(recovery.pathlib.Path, 'stat', autospec=True, side_effect=file_stat), \ + mock.patch.object(recovery.pathlib.Path, 'resolve', autospec=True, side_effect=lambda path, **kw: path), \ + mock.patch.object(recovery.pathlib.Path, 'read_text', autospec=True, side_effect=read_text), \ + mock.patch.object(recovery.os, 'statvfs', return_value=types.SimpleNamespace(f_blocks=19 * 1024**3 // 4096, f_frsize=4096)): + return recovery.filesystem_storage('/synthetic/work', '/synthetic/docker', self.record()['storage']['guest_boot_id']) + + def test_fixed_guest_block_device_bounds_work_and_daemon_storage(self): + self.assertEqual(self.observe_storage(), self.storage_frame()) + + def test_large_shared_device_foreign_mount_or_boot_refuses(self): + for changes in [dict(block_bytes=1024**4), dict(foreign_device=True), + dict(boot='22222222-2222-2222-2222-222222222222'), + dict(mount_text='42 1 8:1 / / rw - overlay overlay rw\n'), + dict(mount_text='42 1 8:1 / / rw - ext4 /dev/sda1 rw\n43 1 8:1 / /share rw - virtiofs host rw\n')]: + with self.subTest(changes=changes), self.assertRaises(SystemExit): + self.observe_storage(**changes) + + def test_storage_authority_is_closed_and_reobserved(self): + storage = self.record()['storage'] + with mock.patch.object(recovery, 'filesystem_storage', return_value=self.storage_frame()): + recovery.validate_storage(storage, '/synthetic/work') + for field, value in [('extra', 1), ('block_bytes', recovery.GUEST_DISK_BYTES + 512), + ('filesystem_bytes', True), ('daemon_id', ''), ('guest_boot_id', 'foreign')]: + bad = dict(storage, **{field: value}) + with self.subTest(field=field), self.assertRaises(SystemExit): + recovery.validate_storage(bad, '/synthetic/work', observe=False) + changed = dict(self.storage_frame(), device='8:2') + with mock.patch.object(recovery, 'filesystem_storage', return_value=changed), \ + self.assertRaisesRegex(SystemExit, 'storage authority changed'): + recovery.validate_storage(storage, '/synthetic/work') + + def test_guest_daemon_images_and_build_cache_must_start_empty(self): + for existing in (None, 'image', 'cache'): + ledger = [] + rows = [{'Type': name, 'TotalCount': '1' if existing == 'cache' and name == 'Build Cache' else '0'} + for name in ('Images', 'Containers', 'Local Volumes', 'Build Cache')] + def command(line): + ledger.append(line) + if line[1] == 'info': + data = json.dumps({'DockerRootDir': '/synthetic/docker', 'ID': 'synthetic-daemon-id'}) + elif line[1] == 'image': + data = 'sha256:existing' if existing == 'image' else '' + elif line[1] == 'system': + data = '\n'.join(json.dumps(row) for row in rows) + else: + data = '' + return types.SimpleNamespace(stdout=data) + args = types.SimpleNamespace(work_dir='/synthetic/work', guest_boot_id=self.record()['storage']['guest_boot_id']) + with mock.patch.object(recovery, 'filesystem_storage', return_value=self.storage_frame()): + if existing: + with self.subTest(existing=existing), self.assertRaises(SystemExit): + recovery.storage_preflight(args, command) + else: + self.assertEqual(recovery.storage_preflight(args, command), self.record()['storage']) + self.assertFalse(any(word in line for line in ledger for word in ('pull', 'create', 'build', 'start', 'rm'))) + if __name__ == '__main__':unittest.main() From bf79bb8c165b4306dcef6670b6f5d7c0a64cf8cd Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:42:19 +0000 Subject: [PATCH 08/14] test: prepare a fixed-disk guest and external guardian proposal --- .../acceptance/condition-history/README.md | 6 +- .../condition-history/guest-host.py | 395 ++++++++++++++++++ .../condition-history/guest-profile.json | 77 ++++ .../condition-history/guest-proposal.md | 155 +++++++ .../acceptance/condition-history/subject.py | 6 +- .../condition-history/test_guest_host.py | 179 ++++++++ .../condition-history/test_packet.py | 2 +- 7 files changed, 815 insertions(+), 5 deletions(-) create mode 100644 scripts/acceptance/condition-history/guest-host.py create mode 100644 scripts/acceptance/condition-history/guest-profile.json create mode 100644 scripts/acceptance/condition-history/guest-proposal.md create mode 100644 scripts/acceptance/condition-history/test_guest_host.py diff --git a/scripts/acceptance/condition-history/README.md b/scripts/acceptance/condition-history/README.md index 38141a6..97bc86c 100644 --- a/scripts/acceptance/condition-history/README.md +++ b/scripts/acceptance/condition-history/README.md @@ -178,8 +178,10 @@ parent and Docker data root. It refuses host/network filesystem shares, a nonempty daemon or build-cache baseline, and a later device/geometry drift. The run record binds that storage identity. Image/cache inventories remain in external-guardian custody until whole-guest/store teardown; the guest never -guesses which layers to delete from a shared daemon. The concrete host guardian -and guest proposal are still being prepared and are not admitted for execution. +guesses which layers to delete from a shared daemon. The concrete +[guest/guardian source proposal](guest-proposal.md) selects cached native Lima/VZ +inputs and a fixed enclosing host disk. Its setup/watchdog and exact admission +prerequisites remain open; no VM execution is admitted. The comparator scans every opened wait, condition outcome and timer row, binding the authored group, member key/fingerprint/occurrence, wait ID and integer timeout. diff --git a/scripts/acceptance/condition-history/guest-host.py b/scripts/acceptance/condition-history/guest-host.py new file mode 100644 index 0000000..768bc25 --- /dev/null +++ b/scripts/acceptance/condition-history/guest-host.py @@ -0,0 +1,395 @@ +"""Source proposal for an external, task-scoped Lima/VZ storage guardian. + +The plan is inert. Guard/audit require a separately admitted private authority +record from fixed-disk setup; they never create or start a VM. +""" +import argparse +import ctypes +import errno +import hashlib +import json +import os +import pathlib +import plistlib +import re +import selectors +import shlex +import stat +import subprocess +import sys +import time + +LIMA_SHA = '216450fc8e3eeb7f50e3030de216c32efeef505b111ed3fb24912a36feebea49' +AGENT_SHA = '4aa75eacc151cbf2b7aec1b9ebe3886a0fc3268977c108308c802ce267c76959' +IMAGE_SHA = '4cd967d2c58aa6971621343d372255b14ffbfe28a185cda0e3c34ac8a1eb15e6' +IMAGE_BYTES = 358_298_593 +STORE_BYTES = 32 * 1024**3 +GUEST_BYTES = 20 * 1024**3 +MEMORY_BYTES = 6 * 1024**3 +PROFILE_SHA = 'fd12240c0a04fbe0614901038586a9cc411f154e11a56d6d1fcd3f4f045f0516' +AUTHORITY_KEYS = {'schema', 'token', 'runner_commit', 'manifest_sha256', 'guardian_sha256', + 'started_unix', 'active_unix', 'cleanup_unix', 'parent', 'limactl', + 'guest_agent', 'host_boot_id', 'image_dev', 'image_ino', 'whole_device', + 'volume_uuid', 'profile_sha256'} + + +def require(ok, label): + if not ok: + raise SystemExit(label) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def canonical(value): + return (json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n').encode() + + +def regular(path, limit=8 * 1024**2): + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_uid == os.getuid(), 'unsafe host handoff') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as stream: + opened = os.fstat(stream.fileno()) + require((opened.st_dev, opened.st_ino) == (before.st_dev, before.st_ino), 'host handoff replaced') + data = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + require(len(data) <= limit and after.st_nlink == 1 + and (opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), 'host handoff changed') + return data + + +def exclusive(path, value): + data = canonical(value) + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_WRONLY, 0o600) + with os.fdopen(fd, 'wb') as stream: + os.fchmod(stream.fileno(), 0o600) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + require(regular(path) == data, 'host handoff changed after write') + return digest(data) + + +def paths(parent, token): + root = pathlib.Path(parent) + require(root.is_absolute() and root.resolve() == root and re.fullmatch('[0-9a-f]{16}', token), 'invalid host scope') + require(root.name == 'sr-' + token and not any(ch.isspace() for ch in str(root)), 'wrong host scope name') + home = root / 'v/l' + name = 'sdk-rust-' + token + require(len(str(home / name / 'ssh.sock.1234567890123456').encode()) < 104, 'Lima socket path is too long') + return {'parent': root, 'mount': root / 'v', 'image': root / 'store.dmg', + 'home': home, 'instance': home / name, 'name': name} + + +def environment(record): + scoped = paths(record['parent'], record['token']) + return {'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'HOME': str(pathlib.Path.home()), + 'LIMA_HOME': str(scoped['home']), + 'LIMA_SHELLENV_BLOCK': '*', 'TMPDIR': str(scoped['mount'] / 'tmp')} + + +def plan(parent, token, limactl, image_path, profile): + scoped = paths(parent, token) + image = pathlib.Path(image_path) + require(image == scoped['mount'] / 'a/guest.raw.gz', 'guest image is outside bounded store') + value = json.loads(json.dumps(profile)) + # Pin the entire JSON document, including provision scripts and networking. + expected = json.dumps(value, sort_keys=True, indent=2) + '\n' + require(digest(expected.encode()) == PROFILE_SHA, 'unreviewed guest template') + require(pathlib.Path(limactl).is_absolute(), 'absolute VM launcher required') + require(value['images'] == [{'location': '__TASK_IMAGE__', 'arch': 'x86_64', 'digest': 'sha256:' + IMAGE_SHA}], 'wrong guest image profile') + require(value['cpus'] == 4 and value['memory'] == '6GiB' and value['disk'] == '20GiB' + and value['vmType'] == 'vz' and value['arch'] == 'x86_64', 'wrong guest resource profile') + require(value['base'] == [] and value['mounts'] == [] and value['additionalDisks'] == [] + and value['copyToHost'] == [] and value['env'] == {} and value['propagateProxyEnv'] is False, + 'inherited inputs or host shares refused') + require(all(value['ssh'][key] is False for key in ('loadDotSSHPubKeys', 'forwardAgent', 'forwardX11', 'forwardX11Trusted')) + and value['containerd'] == {'system': False, 'user': False}, 'forwarded credentials or implicit runtime refused') + require(value['portForwards'] == [{'guestIP': '0.0.0.0', 'guestIPMustBeZero': False, + 'guestPortRange': [1, 65535], 'proto': 'any', 'ignore': True}], 'guest service forwarding refused') + value['images'][0]['location'] = str(image) + return {'schema': 'durable-workflow.condition-guest-plan/v1', 'profile': value, + 'profile_sha256': digest(canonical(value)), 'guest_disk_bytes': GUEST_BYTES, + 'host_store_bytes': STORE_BYTES, 'guest_memory_bytes': MEMORY_BYTES, 'vcpus': 4, + 'minimum_host_available_bytes': 10 * 1024**3, 'minimum_host_free_disk_bytes': 48 * 1024**3, + 'setup_seconds': 300, 'active_seconds': 1740, 'cleanup_seconds': 1800, + 'create_store': ['/usr/bin/hdiutil', 'create', '-sectors', str(STORE_BYTES // 512), + '-type', 'UDIF', '-layout', 'GPTSPUD', '-fs', 'APFS', '-nospotlight', + '-volname', 'sr-' + token, str(scoped['image'])], + 'attach_store': ['/usr/bin/hdiutil', 'attach', '-nobrowse', '-owners', 'on', + '-mountpoint', str(scoped['mount']), '-plist', str(scoped['image'])], + 'create_vm': [limactl, '--tty=false', 'create', '--name', scoped['name'], str(scoped['mount'] / 'guest.yaml')], + 'start_vm': [limactl, '--tty=false', 'start', '--timeout', '5m', scoped['name']], + 'stop_vm': [limactl, '--tty=false', 'stop', '--force', scoped['name']], + 'delete_vm': [limactl, '--tty=false', 'delete', '--force', scoped['name']], + 'environment': environment({'parent': parent, 'token': token})} + + +def call(record, argv): + remaining = record['_cleanup_mono'] - time.monotonic() + require(remaining > 0, 'external cleanup deadline exhausted') + # Read with a hard in-memory ceiling; native commands cannot fill host logs. + child = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=environment(record), close_fds=True) + data = bytearray() + cutoff = time.monotonic() + min(20, remaining) + try: + with selectors.DefaultSelector() as selector: + selector.register(child.stdout, selectors.EVENT_READ) + while selector.get_map(): + require(time.monotonic() < cutoff, 'native scoped control timed out') + for key, _ in selector.select(min(0.1, max(0, cutoff - time.monotonic()))): + block = os.read(key.fileobj.fileno(), 65536) + if not block: + selector.unregister(key.fileobj) + continue + require(len(data) + len(block) <= 2 * 1024**2, 'oversize native control response') + data.extend(block) + child.wait(timeout=max(0.001, cutoff - time.monotonic())) + require(child.returncode == 0, 'native scoped control failed') + return bytes(data) + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=2) + child.stdout.close() + + +def boot_identity(): + value = subprocess.run(['/usr/sbin/sysctl', '-n', 'kern.bootsessionuuid'], check=True, + capture_output=True, text=True, timeout=5).stdout.strip().lower() + require(re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', value), 'missing Darwin boot authority') + return value + + +class BsdInfo(ctypes.Structure): + _fields_ = [(name, ctypes.c_uint32) for name in ( + 'flags', 'status', 'xstatus', 'pid', 'ppid', 'uid', 'gid', 'ruid', 'rgid', 'svuid', 'svgid', 'reserved')] + _fields_ += [('comm', ctypes.c_char * 16), ('name', ctypes.c_char * 32)] + _fields_ += [(name, ctypes.c_uint32) for name in ('nfiles', 'pgid', 'pjobc', 'tdev', 'tpgid')] + _fields_ += [('nice', ctypes.c_int32), ('start_seconds', ctypes.c_uint64), ('start_microseconds', ctypes.c_uint64)] + + +def bsd_identity(pid): + require(sys.platform == 'darwin', 'Darwin host guardian required') + library = ctypes.CDLL('/usr/lib/libproc.dylib', use_errno=True) + library.proc_pidinfo.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64, ctypes.c_void_p, ctypes.c_int] + library.proc_pidinfo.restype = ctypes.c_int + info = BsdInfo() + count = library.proc_pidinfo(pid, 3, 0, ctypes.byref(info), ctypes.sizeof(info)) + if count == 0 and ctypes.get_errno() == errno.ESRCH: + return None + require(count == ctypes.sizeof(info) and info.pid == pid and info.uid == os.getuid() + and info.start_seconds > 0 and info.start_microseconds < 1_000_000, 'unavailable host kernel authority') + return {'pid': pid, 'pgid': info.pgid, 'uid': info.uid, + 'start_seconds': info.start_seconds, 'start_microseconds': info.start_microseconds} + + +def process_identity(pid): + boot = boot_identity() + first, second = bsd_identity(pid), bsd_identity(pid) + require(first == second and boot == boot_identity(), 'host process epoch changed during observation') + return dict(first, boot_id=boot) if first else None + + +def authority(path, expected_sha, source_sha): + raw = regular(path, 64 * 1024) + require(digest(raw) == expected_sha, 'host authority digest mismatch') + value = json.loads(raw) + require(set(value) == AUTHORITY_KEYS and value['schema'] == 'durable-workflow.condition-host/v1', 'wrong host authority schema') + require(value['guardian_sha256'] == source_sha and re.fullmatch('[0-9a-f]{40}', value['runner_commit']) + and re.fullmatch('[0-9a-f]{64}', value['manifest_sha256']), 'wrong captured host source') + require(all(type(value[k]) is int for k in ('started_unix', 'active_unix', 'cleanup_unix', 'image_dev', 'image_ino')) + and value['active_unix'] == value['started_unix'] + 1740 + and value['cleanup_unix'] == value['started_unix'] + 1800, 'wrong host deadlines or inode authority') + require(value['host_boot_id'] == boot_identity(), 'host boot changed') + scoped = paths(value['parent'], value['token']) + info = scoped['parent'].lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() and stat.S_IMODE(info.st_mode) == 0o700, + 'wrong host parent authority') + binary = pathlib.Path(value['limactl']).resolve(strict=True) + require(digest(binary.read_bytes()) == LIMA_SHA, 'reviewed VM launcher changed') + agent = pathlib.Path(value['guest_agent']).resolve(strict=True) + require(digest(agent.read_bytes()) == AGENT_SHA, 'reviewed guest agent changed') + require(re.fullmatch('/dev/disk[0-9]+', value['whole_device']) + and re.fullmatch('[0-9A-Fa-f-]{36}', value['volume_uuid']) + and re.fullmatch('[0-9a-f]{64}', value['profile_sha256']), 'invalid native store authority') + value['_cleanup_mono'] = time.monotonic() + max(0, value['cleanup_unix'] - time.time()) + value['_active_mono'] = time.monotonic() + max(0, value['active_unix'] - time.time()) + return value + + +def verify_store(record): + scoped = paths(record['parent'], record['token']) + info = scoped['image'].lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_uid == os.getuid() + and (info.st_dev, info.st_ino) == (record['image_dev'], record['image_ino']) + and 0 < info.st_size <= STORE_BYTES + 2 * 1024**2, 'fixed host image changed') + entries = plistlib.loads(call(record, ['/usr/bin/hdiutil', 'info', '-plist']))['images'] + matches = [entry for entry in entries if pathlib.Path(entry['image-path']).resolve() == scoped['image']] + require(len(matches) == 1, 'host image attachment is missing or ambiguous') + entities = matches[0]['system-entities'] + require(any(entry.get('dev-entry') == record['whole_device'] for entry in entities) + and any(entry.get('mount-point') == str(scoped['mount']) for entry in entities), 'foreign host image attachment') + whole = plistlib.loads(call(record, ['/usr/sbin/diskutil', 'info', '-plist', record['whole_device']])) + volume = plistlib.loads(call(record, ['/usr/sbin/diskutil', 'info', '-plist', str(scoped['mount'])])) + require(whole['TotalSize'] == STORE_BYTES and volume['VolumeUUID'].lower() == record['volume_uuid'].lower() + and volume['TotalSize'] <= STORE_BYTES and scoped['home'].stat().st_dev == scoped['mount'].stat().st_dev, + 'host store does not have its fixed byte boundary') + require(digest(regular(scoped['mount'] / 'guest.yaml')) == record['profile_sha256'], 'reviewed guest profile changed') + + +def capture_agent(record): + scoped = paths(record['parent'], record['token']) + pid_path = scoped['instance'] / 'ha.pid' + if not pid_path.exists(): + return None + pid_text = regular(pid_path, 64).strip() + require(pid_text.isdigit() and int(pid_text) > 1, 'invalid owned VM PID file') + process = process_identity(int(pid_text)) + require(process and process['start_seconds'] >= record['started_unix'], 'stale VM process epoch') + command = call(record, ['/bin/ps', '-ww', '-p', pid_text.decode(), '-o', 'command=']).decode() + argv = shlex.split(command) + binding = '--pidfile=' + str(pid_path) + pairs = list(zip(argv, argv[1:])) + require(argv and pathlib.Path(argv[0]).resolve() == pathlib.Path(record['limactl']).resolve() + and 'hostagent' in argv and (binding in argv or ('--pidfile', str(pid_path)) in pairs) + and scoped['name'] in argv and process_identity(process['pid']) == process, 'foreign VM host agent') + return process + + +def export_handoff(record, record_sha): + scoped = paths(record['parent'], record['token']) + raw = regular(scoped['parent'] / 'exports.json', 64 * 1024) + value = json.loads(raw) + require(set(value) == {'record_sha256', 'guest_boot_id', 'guest-summary.json', 'guest-audit.json'} + and value['record_sha256'] == record_sha + and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', value['guest_boot_id']), + 'wrong external export authority') + for name in ('guest-summary.json', 'guest-audit.json'): + require(digest(regular(scoped['parent'] / name)) == value[name], 'exported guest proof changed') + return value, digest(raw) + + +def finish_requested(record, record_sha): + scoped = paths(record['parent'], record['token']) + marker = scoped['parent'] / 'finish.json' + if not os.path.lexists(marker): + return False + value = json.loads(regular(marker, 64 * 1024)) + _, export_sha = export_handoff(record, record_sha) + require(value == {'record_sha256': record_sha, 'exports_sha256': export_sha}, 'wrong cleanup request authority') + return True + + +def cleanup(record): + scoped = paths(record['parent'], record['token']) + verify_store(record) + captured_path = scoped['parent'] / 'vm-process.json' + process = json.loads(regular(captured_path)) if captured_path.exists() else capture_agent(record) + if process: + current = process_identity(process['pid']) + if current: + require(current == process and capture_agent(record) == process, + 'VM process epoch changed before scoped stop') + require(process_identity(process['pid']) == process, 'VM process epoch changed at scoped stop') + call(record, [record['limactl'], '--tty=false', 'stop', '--force', scoped['name']]) + require(process_identity(process['pid']) is None, 'owned VM process survived') + require(not (scoped['instance'] / 'ha.pid').exists(), 'VM process custody is uncertain') + if scoped['instance'].exists(): + call(record, [record['limactl'], '--tty=false', 'delete', '--force', scoped['name']]) + require(not scoped['instance'].exists(), 'VM instance storage survived') + verify_store(record) + call(record, ['/usr/bin/hdiutil', 'detach', record['whole_device']]) + require(not os.path.ismount(scoped['mount']), 'owned host store remains mounted') + # A changed inode never gains deletion authority through the pathname. + info = scoped['image'].lstat() + require((info.st_dev, info.st_ino) == (record['image_dev'], record['image_ino']) and info.st_nlink == 1, + 'host image changed before deletion') + scoped['image'].unlink() + return {'guest_absent': True, 'store_unmounted': True, 'image_absent': True, + 'image_cache_disposition': 'whole_task_guest_and_store_removed'} + + +def audit(record, record_sha, source_sha): + scoped = paths(record['parent'], record['token']) + require(not os.path.lexists(scoped['image']) and not os.path.ismount(scoped['mount']) + and not os.path.lexists(scoped['instance']), 'external VM/store survivor') + ready = json.loads(regular(scoped['parent'] / 'host-ready.json')) + require(ready['record_sha256'] == record_sha and ready['source_sha256'] == source_sha + and process_identity(ready['process']['pid']) is None, 'host guardian still active or changed') + vm = json.loads(regular(scoped['parent'] / 'vm-process.json')) + require(process_identity(vm['pid']) is None, 'VM host agent survived') + result = json.loads(regular(scoped['parent'] / 'host-cleanup.json')) + require(result['record_sha256'] == record_sha and result['guardian_sha256'] == source_sha + and result['outcome'] == 'pending_independent_host_audit' + and result['guest_absent'] is True and result['store_unmounted'] is True and result['image_absent'] is True, + 'external cleanup did not pass') + exports, export_sha = export_handoff(record, record_sha) + require(finish_requested(record, record_sha), 'completed guest export request required') + guest = json.loads(regular(scoped['parent'] / 'guest-summary.json')) + guest_audit = json.loads(regular(scoped['parent'] / 'guest-audit.json')) + require(guest['runner_commit'] == record['runner_commit'] and guest['manifest_sha256'] == record['manifest_sha256'] + and guest['checks_outcome'] == 'pass' and guest_audit['outcome'] == 'pending_external_guest_teardown' + and guest['storage']['guest_boot_id'] == exports['guest_boot_id'] + and guest_audit['storage'] == guest['storage'] + and guest_audit['manifest_sha256'] == record['manifest_sha256'] + and guest_audit['summary_sha256'] == exports['guest-summary.json'], 'guest evidence does not authorize acceptance') + result.update(outcome='pass', exports_sha256=export_sha) + exclusive(scoped['parent'] / 'host-audit.json', result) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=('plan', 'guard', 'audit')) + parser.add_argument('--record') + parser.add_argument('--record-sha256') + parser.add_argument('--plan-input') + args = parser.parse_args(argv) + if args.mode == 'plan': + value = json.loads(pathlib.Path(args.plan_input).read_bytes()) + print(json.dumps(plan(**value), sort_keys=True)) + return + require('-I' in sys.orig_argv and '-S' in sys.orig_argv and '-c' in sys.orig_argv, 'captured host guardian required') + source_sha = digest(sys.orig_argv[sys.orig_argv.index('-c') + 1].encode()) + record = authority(pathlib.Path(args.record), args.record_sha256, source_sha) + scoped = paths(record['parent'], record['token']) + if args.mode == 'guard': + require(record['started_unix'] <= time.time() < record['started_unix'] + 300, + 'host setup cutoff exceeded') + verify_store(record) + exclusive(scoped['parent'] / 'host-ready.json', {'record_sha256': args.record_sha256, + 'source_sha256': source_sha, 'process': process_identity(os.getpid())}) + control_error = None + try: + while time.monotonic() < record['_active_mono']: + process = capture_agent(record) + receipt = scoped['parent'] / 'vm-process.json' + if process and not receipt.exists(): + exclusive(receipt, process) + if finish_requested(record, args.record_sha256): + break + time.sleep(0.25) + except BaseException as error: + # Bad handoffs or observations never bypass the cleanup attempt. + control_error = type(error).__name__ + try: + result = dict(cleanup(record), outcome='pending_independent_host_audit') + except BaseException as error: + result = {'outcome': 'fail', 'error_type': type(error).__name__, + 'custody': 'external_owner_must_resolve_retained_task_objects'} + if control_error: + result.update(outcome='fail', control_error_type=control_error) + result.update(record_sha256=args.record_sha256, guardian_sha256=source_sha) + exclusive(scoped['parent'] / 'host-cleanup.json', result) + require(result['outcome'] != 'fail', 'external cleanup incomplete') + else: + print(json.dumps(audit(record, args.record_sha256, source_sha))) + + +if __name__ == '__main__': + main() diff --git a/scripts/acceptance/condition-history/guest-profile.json b/scripts/acceptance/condition-history/guest-profile.json new file mode 100644 index 0000000..6129427 --- /dev/null +++ b/scripts/acceptance/condition-history/guest-profile.json @@ -0,0 +1,77 @@ +{ + "additionalDisks": [], + "arch": "x86_64", + "base": [], + "caCerts": { + "certs": [], + "files": [], + "removeDefaults": false + }, + "containerd": { + "system": false, + "user": false + }, + "copyToHost": [], + "cpus": 4, + "disk": "20GiB", + "env": {}, + "hostResolver": { + "enabled": false, + "hosts": {} + }, + "images": [ + { + "arch": "x86_64", + "digest": "sha256:4cd967d2c58aa6971621343d372255b14ffbfe28a185cda0e3c34ac8a1eb15e6", + "location": "__TASK_IMAGE__" + } + ], + "memory": "6GiB", + "minimumLimaVersion": "2.2.1", + "mountInotify": false, + "mountType": "virtiofs", + "mounts": [], + "nestedVirtualization": false, + "networks": [ + { + "vzNAT": true + } + ], + "portForwards": [ + { + "guestIP": "0.0.0.0", + "guestIPMustBeZero": false, + "guestPortRange": [ + 1, + 65535 + ], + "ignore": true, + "proto": "any" + } + ], + "propagateProxyEnv": false, + "provision": [ + { + "mode": "dependency", + "script": "#!/bin/sh\nset -eu\ncommand -v docker >/dev/null\ncommand -v python3 >/dev/null\ncommand -v git >/dev/null\n", + "skipDefaultDependencyResolution": true + } + ], + "ssh": { + "forwardAgent": false, + "forwardX11": false, + "forwardX11Trusted": false, + "loadDotSSHPubKeys": false, + "localPort": 0 + }, + "timezone": "UTC", + "upgradePackages": false, + "user": { + "home": "/home/fixture.guest", + "name": "fixture", + "passwordlessSudo": true, + "shell": "/bin/bash", + "uid": 1000 + }, + "vmType": "vz" +} diff --git a/scripts/acceptance/condition-history/guest-proposal.md b/scripts/acceptance/condition-history/guest-proposal.md new file mode 100644 index 0000000..3a40c2d --- /dev/null +++ b/scripts/acceptance/condition-history/guest-proposal.md @@ -0,0 +1,155 @@ +# Disposable guest source proposal + +This is a source proposal for #210, not an execution admission. No VM, disk +image, pull or consumer build has been started. Security must review the exact +source and General must admit a private invocation with fresh capacity before +any of the commands below can run. The shared-daemon proposal is superseded. + +The selected backend is the already installed Lima 2.2.1 native Virtualization +framework backend on an Intel macOS 15.8.2 host. It avoids a host installation, +provider account or paid resource. An existing writable shared VM is unsuitable +because it shares the host home and owns unrelated images and caches. The two +Linux hosts have neither an installed VM launcher nor current-user KVM access. +The other native VZ host is arm64, while this acceptance pins amd64 manifests. +Private host names, cache paths, scope and capacity receipts belong in the +execution proposal rather than this repository. + +## Immutable inputs and cost + +| Input | Exact identity | +| --- | --- | +| `limactl` | Lima 2.2.1, 35,175,712 bytes, SHA-256 `216450fc8e3eeb7f50e3030de216c32efeef505b111ed3fb24912a36feebea49` | +| Linux amd64 guest agent gzip | 8,257,812 bytes, SHA-256 `4aa75eacc151cbf2b7aec1b9ebe3886a0fc3268977c108308c802ce267c76959` | +| Cached guest image | [colima-core v0.10.4 Ubuntu 24.04 Docker amd64 raw gzip](https://github.com/abiosoft/colima-core/releases/download/v0.10.4/ubuntu-24.04-minimal-cloudimg-amd64-docker.raw.gz), 358,298,593 bytes, SHA-256 `4cd967d2c58aa6971621343d372255b14ffbfe28a185cda0e3c34ac8a1eb15e6` | +| Native tools | macOS 15.8.2 build 24H232 `hdiutil`, `diskutil`, `sysctl`, public `libproc` process information | +| Host guardian | Captured `guest-host.py` Git blob at admitted R, SHA-256 in the R manifest and private authority | +| Guest profile | Complete `guest-profile.json` at R, with its one image path replaced by the task-local path | + +The cache was hashed read-only and matches the public release asset digest and +size. No download was needed. The release source builds Docker 29.5.2, but that +is an expectation to verify inside the fresh guest, not an observed engine +receipt. Python 3, Git, Docker, a block-backed ext4 root and an empty daemon/cache +are required. Missing dependencies cause refusal without package installation. + +The bounded alternative costs **zero provider dollars**, one 4-vCPU / 6-GiB +guest and one fixed **32-GiB** read/write UDIF containing the complete fresh Lima +home, inputs, VM disk, generated SSH keys, temporary files and logs. The guest +disk is **20 GiB**, including its OS, Cargo cache, work, Docker/containerd stores, +images and logs. Its block-device and filesystem ceilings are checked before +registry requests and again before daemon mutations. Space exhaustion fails the +single attempt; there is no enlargement, retry, shared-cache reuse or pruning. +Only two sanitized guest results, each at most 8 MiB, plus bounded private host +authority/receipt files are exported outside the UDIF. Native control output is +held in memory with a combined 2-MiB ceiling and never streamed to unbounded logs. + +Admission requires at least **10 GiB available RAM** and **48 GiB free host disk**, +fresh metrics, and agreement from the owners of any existing shared VM. The +observed host had about 6.5 GiB available, so it did not pass. A coordinated idle +window could release the existing 6-GiB shared VM allocation; this proposal does +not authorize stopping it. The fixed disk is additional to that shared VM, which +is never an acceptance input. No global daemon setting or host package changes. + +The entire invocation starts at S, active work ends at S+1740 seconds, and all +cleanup ends by S+1800. Setup consumes the same window and must finish within +300 seconds. Host guardian timers use monotonic time once the externally captured +absolute cutoffs are bound. The existing guest Cell keeps its 600-second bound, +two Cargo jobs, sequential build/runtime phases and per-role caps. + +## Launch and custody stages + +`guest-host.py plan --plan-input PRIVATE_JSON` is inert. Its closed template check +produces the exact native argument vectors, environment, rendered profile digest +and resource/deadline budgets. It accepts private canonical parent P named +`sr-T`, where T is a fresh 16-hex token. Its Lima home is `P/v/l`, guest name +`sdk-rust-T`, gzip `P/v/a/guest.raw.gz` and fixed image `P/store.dmg`. The short +private path must satisfy Lima's Unix socket limit. No shell interpolation is +used for native controls. + +1. Before the first mutation, an external setup owner captures R/M, S and both + cutoffs, input/tool hashes, architecture, OS, native Python version, free space + and current capacity. It creates an exclusive owner-only P and records intent. + An outside-guest setup watchdog owns the 300-second setup cutoff and partial + image/attachment custody. This source packet does not implement that setup + watchdog: its captured source and precise invocation are a remaining admission + prerequisite, even if every inert control passes. +2. Use only the plan's `hdiutil create -sectors 67108864 -type UDIF -layout + GPTSPUD -fs APFS -nospotlight` and `attach -nobrowse -owners on -mountpoint P/v + -plist` commands. Capture image device/inode, the whole `/dev/diskN`, APFS volume + UUID and the 32-GiB whole-device readback. Refuse an existing image, attachment, + mount, Lima home or instance. Copy the already verified public gzip and guest + agent into the store without networking, check hashes again, and write the + rendered JSON-as-YAML profile exclusively. No private host source or credential + tree is copied. Local image input and disabled containerd downloads avoid + Lima's global downloader cache. +3. Write an exclusive 0600 authority record with schema + `durable-workflow.condition-host/v1`. Its exact fields are `AUTHORITY_KEYS` in + `guest-host.py`: R/M, guardian digest, token, host boot UUID, S/cutoffs, private + scope and launcher/agent paths, image device/inode, whole device, volume UUID + and rendered-profile digest. Capture its SHA-256 outside the store. The + external setup watchdog retains custody until the guardian is ready. +4. Start **captured Git bytes**, not a mutable script path, with native Python + `-I -S -c`, mode `guard`, `--record` and `--record-sha256`. The guardian verifies + launcher/agent digests, host boot, authority, image inode/geometry, attachment + and profile. Its exclusive `host-ready.json` binds authority/source and the + guardian's kernel process epoch. Only after independent readiness verification + may the setup owner run the exact scoped Lima `create` and `start` vectors. + The VM must not start before this custody transfer. Setup failure before that + transfer remains the setup watchdog's responsibility. +5. Lima receives a fresh `LIMA_HOME`, a minimal environment, and no SSH agent, + Docker context, proxy or cloud environment. Its host HOME remains the actual + account home, with host SSH-key loading disabled. The complete profile has no + base template, host mount, copied CA, copied host file, package install, service + forward or nested virtualization. VZ NAT permits public package/image fetches. + Only Lima's generated guest-control SSH connection is permitted. It does not + forward host credentials. `hostResolver` and proxy propagation are disabled. +6. The guardian captures the task instance's `ha.pid` only after binding its + scoped hostagent command and native kernel process epoch. Public + `proc_pidinfo(PROC_PIDTBSDINFO)` provides start seconds **and microseconds**, + UID, PID and group; consistent double reads plus host boot UUID are required. + There is no `ps lstart` signal authority. Revalidate immediately before scoped + `limactl stop --force`. The guardian never sends a numeric host kill signal to + a recovered PID. Missing, changed or foreign authority is a failed result + requiring the external owner, not permission to stop another instance. +7. Obtain the guest's Linux boot ID independently over the generated SSH control + path. Transfer a clean public R checkout with the immutable H tag and invoke + the README's captured entry with that B and the same S/cutoffs. No host source + mount or credential is used. The guest requires an empty daemon/build cache + and ext4 root/work/Docker data on its fixed block device before pulling. + Existing Linux boot/start-tick recovery and full internal-network readbacks + are preserved. Fetch is public/anonymous; credential tests use synthetic values. +8. After guest owner and recovery exit, independently run the captured guest + audit, require the sanitized summary and its exact SHA, and export only the + two bounded JSON results to P. An external exclusive `exports.json` binds the + host record digest, independently observed `guest_boot_id` and both file + digests. An exclusive `finish.json` binds the host record and exact exports + digest. A missing, invalid or changed handoff cannot produce acceptance. +9. At a valid finish request or the unchanged active cutoff, the guardian stops + only the bound scoped instance, confirms the hostagent absent, deletes the + scoped instance, rechecks the enclosing image, detaches its bound whole device, + confirms the mount absent and unlinks only the original image inode. Images, + caches, work, logs and ephemeral keys disappear together. Cleanup never prunes + a shared daemon. It writes exclusive `host-cleanup.json` outside the disk. +10. After guardian exit, run the same captured source in independent `audit` + mode. Require image/mount/instance absence, guardian and VM hostagent absence, + exact host custody/source receipts, R/M/B agreement, guest audit and summary + digest agreement. Its exclusive `host-audit.json` may then record pass. A + guest pass alone remains `pending_external_guest_teardown`. + +Failure at any stage is terminal for this attempt. The external setup owner +handles partial create/attach failures under its setup watchdog. The guardian +attempts cleanup after malformed exports or failed process observation. If a +device, inode, PID epoch or mount changed, it refuses destructive guesses and +records retained custody for the external owner, who must resolve it before any +capacity release. Exhausted time, a failed detach or a surviving process/store +is failure, even when product checks passed. A hard host reboot is an external +recovery case, not an unattended-success claim. + +## Remaining review gates + +This packet intentionally cannot create/start a guest by itself. Before asking +for execution, supply the private setup/watchdog source and exact handoff, verify +the native APFS/Lima readback schema and guest dependency expectations, and obtain +fresh sufficient capacity. Security reviews the full executable/storage packet; +General admits its exact R/M/command/window. Any admitted source change requires +new exact review. The old missing #191 raw capture and PHP-only #192 producer +provenance remain explicit; this guest does not alter either acceptance boundary. diff --git a/scripts/acceptance/condition-history/subject.py b/scripts/acceptance/condition-history/subject.py index e45cc66..754bb76 100644 --- a/scripts/acceptance/condition-history/subject.py +++ b/scripts/acceptance/condition-history/subject.py @@ -15,11 +15,13 @@ H = '4b2440a5618ccc1c98960ea2e1e62127042c3590' RUNTIME_FILES = tuple(PREFIX + p for p in ( 'run.py', 'subject.py', 'engine.py', 'recover.py', 'resources.py', - 'role-supervisor.sh', 'check_history.py', 'fixture/Cargo.toml', 'fixture/tests/conditions.rs')) + ( + 'role-supervisor.sh', 'check_history.py', 'guest-host.py', 'guest-profile.json', + 'fixture/Cargo.toml', 'fixture/tests/conditions.rs')) + ( 'scripts/acceptance/installed-native/run.py', 'scripts/acceptance/installed-native/cell-supervisor.sh') PACKET_FILES = {p for p in RUNTIME_FILES if p.startswith(PREFIX)} | { - PREFIX + p for p in ('README.md', 'test_packet.py', 'test_subject.py', 'test_resources.py', 'test_audit.py')} + PREFIX + p for p in ('README.md', 'guest-proposal.md', 'test_guest_host.py', + 'test_packet.py', 'test_subject.py', 'test_resources.py', 'test_audit.py')} MODULES = {'engine': PREFIX + 'engine.py', 'resources': PREFIX + 'resources.py', 'native': 'scripts/acceptance/installed-native/run.py', 'comparator': PREFIX + 'check_history.py', 'recovery': PREFIX + 'recover.py'} diff --git a/scripts/acceptance/condition-history/test_guest_host.py b/scripts/acceptance/condition-history/test_guest_host.py new file mode 100644 index 0000000..7dd41ef --- /dev/null +++ b/scripts/acceptance/condition-history/test_guest_host.py @@ -0,0 +1,179 @@ +"""External guest controls use fake native commands. No VM or disk is created.""" +import copy +import ctypes +import json +import os +import pathlib +import shutil +import sys +import tempfile +import time +import unittest +from unittest import mock +from test_subject import HERE, load + +host = load('condition_guest_host_tests', HERE / 'guest-host.py') +TOKEN = '0123456789abcdef' +BOOT = '11111111-1111-1111-1111-111111111111' + + +class GuestHost(unittest.TestCase): + def setUp(self): + self.temp = pathlib.Path(tempfile.mkdtemp(dir='/tmp', prefix='sr-')).resolve() + self.addCleanup(shutil.rmtree, self.temp) + self.parent = self.temp / ('sr-' + TOKEN) + self.parent.mkdir(mode=0o700) + self.record = {'parent': str(self.parent), 'token': TOKEN, 'limactl': '/reviewed/limactl', + 'runner_commit': 'a' * 40, 'manifest_sha256': 'b' * 64, 'guardian_sha256': 'c' * 64, + 'started_unix': int(time.time()), 'active_unix': int(time.time()) + 1740, + 'cleanup_unix': int(time.time()) + 1800, '_cleanup_mono': time.monotonic() + 1800} + self.scoped = host.paths(str(self.parent), TOKEN) + self.profile = json.loads((HERE / 'guest-profile.json').read_bytes()) + + def put(self, name, value): + path = self.parent / name + path.write_bytes(host.canonical(value)) + return host.digest(path.read_bytes()) + + def proposal(self, profile=None): + return host.plan(str(self.parent), TOKEN, self.record['limactl'], + str(self.scoped['mount'] / 'a/guest.raw.gz'), profile or self.profile) + + def test_plan_uses_fixed_store_and_fresh_lima_home(self): + with mock.patch.object(host.subprocess, 'Popen', side_effect=AssertionError('inert plan')): + value = self.proposal() + self.assertEqual(value['host_store_bytes'], 32 * 1024**3) + self.assertEqual(value['guest_disk_bytes'], 20 * 1024**3) + self.assertEqual(value['create_store'][2:4], ['-sectors', '67108864']) + self.assertEqual(value['environment']['LIMA_HOME'], str(self.scoped['home'])) + self.assertNotIn('SSH_AUTH_SOCK', value['environment']) + self.assertNotIn('DOCKER_HOST', value['environment']) + self.assertNotIn('HTTP_PROXY', value['environment']) + self.assertEqual(value['profile']['mounts'], []) + + def test_any_template_change_is_a_new_source_subject(self): + for key, value in {'mounts': [{'location': '~'}], 'base': ['template:default'], + 'disk': '21GiB', 'cpus': 8, 'networks': [], 'provision': [], + 'propagateProxyEnv': True, 'portForwards': [], + 'ssh': dict(self.profile['ssh'], forwardAgent=True)}.items(): + candidate = copy.deepcopy(self.profile) + candidate[key] = value + with self.subTest(key=key), self.assertRaises(SystemExit): + self.proposal(candidate) + + def test_foreign_scope_and_unbounded_image_paths_refuse(self): + for root, token, image in [('/tmp/foreign', TOKEN, '/tmp/image'), + (str(self.parent), 'foreign', '/tmp/image'), + (str(self.parent), TOKEN, '/tmp/image')]: + with self.subTest(root=root, token=token), self.assertRaises(SystemExit): + host.plan(root, token, '/reviewed/limactl', image, self.profile) + + def test_public_darwin_struct_and_consistent_kernel_epoch(self): + self.assertEqual(ctypes.sizeof(host.BsdInfo), 136) + process = {'pid': 1234, 'pgid': 1234, 'uid': os.getuid(), + 'start_seconds': 1000, 'start_microseconds': 123456} + with mock.patch.object(host, 'boot_identity', return_value=BOOT), \ + mock.patch.object(host, 'bsd_identity', return_value=process): + self.assertEqual(host.process_identity(1234), dict(process, boot_id=BOOT)) + for changed in (dict(process, start_microseconds=123457), None): + with mock.patch.object(host, 'boot_identity', return_value=BOOT), \ + mock.patch.object(host, 'bsd_identity', side_effect=[process, changed]), self.assertRaises(SystemExit): + host.process_identity(1234) + with mock.patch.object(host, 'boot_identity', side_effect=[BOOT, 'other']), \ + mock.patch.object(host, 'bsd_identity', return_value=process), self.assertRaises(SystemExit): + host.process_identity(1234) + + def exports(self): + storage = {'guest_boot_id': BOOT} + summary_sha = self.put('guest-summary.json', {'runner_commit': self.record['runner_commit'], + 'manifest_sha256': self.record['manifest_sha256'], 'checks_outcome': 'pass', 'storage': storage}) + audit_sha = self.put('guest-audit.json', {'outcome': 'pending_external_guest_teardown', + 'manifest_sha256': self.record['manifest_sha256'], 'summary_sha256': summary_sha, 'storage': storage}) + exports_sha = self.put('exports.json', {'record_sha256': 'd' * 64, 'guest_boot_id': BOOT, + 'guest-summary.json': summary_sha, 'guest-audit.json': audit_sha}) + self.put('finish.json', {'record_sha256': 'd' * 64, 'exports_sha256': exports_sha}) + + def test_finish_requires_exact_external_exports_and_digests(self): + self.assertFalse(host.finish_requested(self.record, 'd' * 64)) + self.exports() + self.assertTrue(host.finish_requested(self.record, 'd' * 64)) + self.put('finish.json', {'record_sha256': 'foreign', 'exports_sha256': 'e' * 64}) + with self.assertRaises(SystemExit):host.finish_requested(self.record, 'd' * 64) + self.exports() + self.put('guest-summary.json', {'checks_outcome': 'pass'}) + with self.assertRaises(SystemExit):host.finish_requested(self.record, 'd' * 64) + + def audit_files(self): + self.exports() + self.put('host-ready.json', {'record_sha256': 'd' * 64, 'source_sha256': 'c' * 64, + 'process': {'pid': 1234}}) + self.put('vm-process.json', {'pid': 2345}) + self.put('host-cleanup.json', {'record_sha256': 'd' * 64, 'guardian_sha256': 'c' * 64, + 'outcome': 'pending_independent_host_audit', 'guest_absent': True, + 'store_unmounted': True, 'image_absent': True}) + + def test_independent_host_audit_requires_guest_proof_and_both_processes_absent(self): + self.audit_files() + with mock.patch.object(host, 'process_identity', return_value=None): + self.assertEqual(host.audit(self.record, 'd' * 64, 'c' * 64)['outcome'], 'pass') + with mock.patch.object(host, 'process_identity', return_value=None), self.assertRaises(FileExistsError): + host.audit(self.record, 'd' * 64, 'c' * 64) + (self.parent / 'host-audit.json').unlink() + for survivor in (1234, 2345): + def observe(pid):return {'pid': pid} if pid == survivor else None + with self.subTest(survivor=survivor), mock.patch.object(host, 'process_identity', side_effect=observe), \ + self.assertRaises(SystemExit):host.audit(self.record, 'd' * 64, 'c' * 64) + self.scoped['image'].write_bytes(b'survivor') + with self.assertRaises(SystemExit):host.audit(self.record, 'd' * 64, 'c' * 64) + + def test_host_audit_refuses_rebound_guest_and_forged_cleanup(self): + self.audit_files() + for filename, key, value in [('guest-summary.json', 'runner_commit', 'e' * 40), + ('guest-audit.json', 'outcome', 'pass'), ('host-cleanup.json', 'guardian_sha256', 'foreign')]: + self.audit_files() + path = self.parent / filename + data = json.loads(path.read_bytes());data[key] = value + self.put(filename, data) + with self.subTest(filename=filename), mock.patch.object(host, 'process_identity', return_value=None), \ + self.assertRaises(SystemExit):host.audit(self.record, 'd' * 64, 'c' * 64) + + def test_native_control_stream_has_hard_output_and_time_bounds(self): + # This launches only an inert Python child, with neither native VM tools nor network. + result = host.call(self.record, [sys.executable, '-I', '-S', '-c', 'print("bounded")']) + self.assertEqual(result, b'bounded\n') + with self.assertRaisesRegex(SystemExit, 'oversize'): + host.call(self.record, [sys.executable, '-I', '-S', '-c', 'import os;os.write(1,b"x"*2200000)']) + self.record['_cleanup_mono'] = time.monotonic() - 1 + with mock.patch.object(host.subprocess, 'Popen', side_effect=AssertionError('deadline before launch')), \ + self.assertRaisesRegex(SystemExit, 'deadline'): + host.call(self.record, ['/never/launch']) + + def test_foreign_store_refuses_all_cleanup_mutations(self): + with mock.patch.object(host, 'verify_store', side_effect=SystemExit('foreign volume')), \ + mock.patch.object(host, 'call', side_effect=AssertionError('must not mutate')), self.assertRaises(SystemExit): + host.cleanup(self.record) + + def test_changed_process_epoch_refuses_scoped_stop(self): + self.put('vm-process.json', {'pid': 2345, 'start_microseconds': 1}) + with mock.patch.object(host, 'verify_store'), \ + mock.patch.object(host, 'process_identity', return_value={'pid': 2345, 'start_microseconds': 2}), \ + mock.patch.object(host, 'call', side_effect=AssertionError('must not stop')), self.assertRaises(SystemExit): + host.cleanup(self.record) + + def test_bad_guard_observation_still_attempts_cleanup(self): + self.record['_active_mono'] = time.monotonic() + 20 + result = {'guest_absent': True, 'store_unmounted': True, 'image_absent': True} + with mock.patch.object(host.sys, 'orig_argv', ['python', '-I', '-S', '-c', 'captured']), \ + mock.patch.object(host, 'authority', return_value=self.record), mock.patch.object(host, 'verify_store'), \ + mock.patch.object(host, 'process_identity', return_value={'pid': 1234}), \ + mock.patch.object(host, 'capture_agent', side_effect=SystemExit('uncertain')), \ + mock.patch.object(host, 'cleanup', return_value=result) as cleanup, self.assertRaises(SystemExit): + host.main(['guard', '--record', '/unused', '--record-sha256', 'd' * 64]) + cleanup.assert_called_once_with(self.record) + receipt = json.loads((self.parent / 'host-cleanup.json').read_bytes()) + self.assertEqual(receipt['outcome'], 'fail') + self.assertEqual(receipt['control_error_type'], 'SystemExit') + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/acceptance/condition-history/test_packet.py b/scripts/acceptance/condition-history/test_packet.py index 21f9f7d..32f9a5d 100644 --- a/scripts/acceptance/condition-history/test_packet.py +++ b/scripts/acceptance/condition-history/test_packet.py @@ -47,7 +47,7 @@ def test_wrong_h_refuses_under_python_optimization(self): self.assertFalse(pathlib.Path(self.args.work_dir).exists()) def test_no_optimizable_runtime_assert(self): - for name in ('run.py','subject.py','engine.py','resources.py','recover.py'): + for name in ('run.py','subject.py','engine.py','resources.py','recover.py','guest-host.py'): with self.subTest(name=name): self.assertFalse(any(isinstance(n,ast.Assert) for n in ast.walk(ast.parse((HERE/name).read_text())))) From b5cf1e9df9b3ffe88e3fa3290074ae10a8422171 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 03:01:38 +0000 Subject: [PATCH 09/14] test: put fixed-disk setup under the resident guest watchdog --- .../acceptance/condition-history/README.md | 5 +- .../condition-history/guest-host.py | 219 ++++++++++++++++-- .../condition-history/guest-proposal.md | 77 +++--- .../condition-history/test_guest_host.py | 111 +++++++++ 4 files changed, 367 insertions(+), 45 deletions(-) diff --git a/scripts/acceptance/condition-history/README.md b/scripts/acceptance/condition-history/README.md index 97bc86c..507d561 100644 --- a/scripts/acceptance/condition-history/README.md +++ b/scripts/acceptance/condition-history/README.md @@ -180,8 +180,9 @@ The run record binds that storage identity. Image/cache inventories remain in external-guardian custody until whole-guest/store teardown; the guest never guesses which layers to delete from a shared daemon. The concrete [guest/guardian source proposal](guest-proposal.md) selects cached native Lima/VZ -inputs and a fixed enclosing host disk. Its setup/watchdog and exact admission -prerequisites remain open; no VM execution is admitted. +inputs and a fixed enclosing host disk. The captured external guardian also owns +bounded setup before allocation. Exact source review, private handoff and fresh +capacity remain prerequisites; no VM execution is admitted. The comparator scans every opened wait, condition outcome and timer row, binding the authored group, member key/fingerprint/occurrence, wait ID and integer timeout. diff --git a/scripts/acceptance/condition-history/guest-host.py b/scripts/acceptance/condition-history/guest-host.py index 768bc25..02386b6 100644 --- a/scripts/acceptance/condition-history/guest-host.py +++ b/scripts/acceptance/condition-history/guest-host.py @@ -1,7 +1,7 @@ """Source proposal for an external, task-scoped Lima/VZ storage guardian. -The plan is inert. Guard/audit require a separately admitted private authority -record from fixed-disk setup; they never create or start a VM. +The plan is inert. Prepare allocates native storage only after separate admission; +guard/audit consume captured private authority. No mode creates or starts a VM. """ import argparse import ctypes @@ -31,6 +31,7 @@ 'started_unix', 'active_unix', 'cleanup_unix', 'parent', 'limactl', 'guest_agent', 'host_boot_id', 'image_dev', 'image_ino', 'whole_device', 'volume_uuid', 'profile_sha256'} +INTENT_KEYS = (AUTHORITY_KEYS - {'image_dev', 'image_ino', 'whole_device', 'volume_uuid', 'profile_sha256'}) | {'cached_image'} def require(ok, label): @@ -88,7 +89,7 @@ def environment(record): scoped = paths(record['parent'], record['token']) return {'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'HOME': str(pathlib.Path.home()), 'LIMA_HOME': str(scoped['home']), - 'LIMA_SHELLENV_BLOCK': '*', 'TMPDIR': str(scoped['mount'] / 'tmp')} + 'LIMA_SHELLENV_BLOCK': '*', 'TMPDIR': record.get('_setup_tmp', str(scoped['mount'] / 'tmp'))} def plan(parent, token, limactl, image_path, profile): @@ -129,13 +130,13 @@ def plan(parent, token, limactl, image_path, profile): def call(record, argv): - remaining = record['_cleanup_mono'] - time.monotonic() + remaining = min(record['_cleanup_mono'], record.get('_stage_mono', float('inf'))) - time.monotonic() require(remaining > 0, 'external cleanup deadline exhausted') # Read with a hard in-memory ceiling; native commands cannot fill host logs. child = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=environment(record), close_fds=True) data = bytearray() - cutoff = time.monotonic() + min(20, remaining) + cutoff = time.monotonic() + min(record.get('_command_seconds', 20), remaining) try: with selectors.DefaultSelector() as selector: selector.register(child.stdout, selectors.EVENT_READ) @@ -222,7 +223,7 @@ def authority(path, expected_sha, source_sha): return value -def verify_store(record): +def verify_store(record, profile_required=True): scoped = paths(record['parent'], record['token']) info = scoped['image'].lstat() require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_uid == os.getuid() @@ -237,9 +238,190 @@ def verify_store(record): whole = plistlib.loads(call(record, ['/usr/sbin/diskutil', 'info', '-plist', record['whole_device']])) volume = plistlib.loads(call(record, ['/usr/sbin/diskutil', 'info', '-plist', str(scoped['mount'])])) require(whole['TotalSize'] == STORE_BYTES and volume['VolumeUUID'].lower() == record['volume_uuid'].lower() - and volume['TotalSize'] <= STORE_BYTES and scoped['home'].stat().st_dev == scoped['mount'].stat().st_dev, + and volume['TotalSize'] <= STORE_BYTES and volume['MountPoint'] == str(scoped['mount']) + and volume['FilesystemType'] == 'apfs', 'host store does not have its fixed byte boundary') - require(digest(regular(scoped['mount'] / 'guest.yaml')) == record['profile_sha256'], 'reviewed guest profile changed') + if profile_required: + require(scoped['home'].stat().st_dev == scoped['mount'].stat().st_dev + and digest(regular(scoped['mount'] / 'guest.yaml')) == record['profile_sha256'], + 'reviewed guest profile changed') + + +def input_bytes(path, expected_size, expected_sha, destination=None, deadline=None): + """Hash/copy only the pinned public inputs, streaming within the setup bound.""" + path = pathlib.Path(path).resolve(strict=True) + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_size == expected_size, + 'wrong public input size or type') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + stream = os.fdopen(fd, 'rb') + target = None + state, count = hashlib.sha256(), 0 + try: + if destination: + target = os.fdopen(os.open(destination, os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_WRONLY, 0o600), 'wb') + with stream: + opened = os.fstat(stream.fileno()) + require((opened.st_dev, opened.st_ino) == (before.st_dev, before.st_ino), 'public input replaced') + while True: + if deadline is not None:require(time.monotonic() < deadline, 'public input setup cutoff') + block = stream.read(1024**2) + if not block:break + count += len(block) + require(count <= expected_size, 'public input grew') + state.update(block) + if target:target.write(block) + after = os.fstat(stream.fileno()) + require((opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns) and after.st_nlink == 1, + 'public input changed') + require(count == expected_size and state.hexdigest() == expected_sha, 'public input digest mismatch') + if target: + target.flush() + os.fsync(target.fileno()) + finally: + if not stream.closed:stream.close() + if target:target.close() + + +def setup_intent(path, expected_sha, source_sha): + require(stat.S_IMODE(path.lstat().st_mode) == 0o600, 'private setup intent required') + raw = regular(path, 64 * 1024) + require(digest(raw) == expected_sha, 'setup intent digest mismatch') + value = json.loads(raw) + require(set(value) == INTENT_KEYS and value['schema'] == 'durable-workflow.condition-host-setup/v1', + 'wrong setup intent schema') + require(value['guardian_sha256'] == source_sha and re.fullmatch('[0-9a-f]{40}', value['runner_commit']) + and re.fullmatch('[0-9a-f]{64}', value['manifest_sha256']), 'wrong setup source authority') + require(all(type(value[k]) is int for k in ('started_unix', 'active_unix', 'cleanup_unix')) + and value['active_unix'] == value['started_unix'] + 1740 + and value['cleanup_unix'] == value['started_unix'] + 1800 + and value['started_unix'] <= time.time() < value['started_unix'] + 300, 'wrong setup deadlines') + require(value['host_boot_id'] == boot_identity(), 'setup host boot changed') + scoped = paths(value['parent'], value['token']) + require(not os.path.lexists(scoped['parent']), 'setup scope already exists') + for key in ('limactl', 'guest_agent', 'cached_image'): + require(pathlib.Path(value[key]).is_absolute(), 'absolute pinned input required') + now, monotonic = time.time(), time.monotonic() + value['_cleanup_mono'] = monotonic + max(0, value['cleanup_unix'] - now) + value['_active_mono'] = monotonic + max(0, value['active_unix'] - now) + value['_stage_mono'] = monotonic + max(0, value['started_unix'] + 300 - now) + return value + + +def capacity(record): + require(sys.platform == 'darwin' and os.uname().machine == 'x86_64', 'native Intel VZ host required') + version = call(record, ['/usr/bin/sw_vers', '-productVersion']).decode().strip() + build = call(record, ['/usr/bin/sw_vers', '-buildVersion']).decode().strip() + require((version, build) == ('15.8.2', '24H232'), 'reviewed native OS changed') + memory = call(record, ['/usr/bin/vm_stat']).decode() + page = re.search(r'page size of (\d+) bytes', memory) + require(page, 'missing host page geometry') + available = 0 + for label in ('Pages free', 'Pages inactive', 'Pages speculative'): + match = re.search(r'^' + label + r':\s+(\d+)\.', memory, re.MULTILINE) + require(match, 'missing host memory readback') + available += int(match[1]) * int(page[1]) + filesystem = os.statvfs(paths(record['parent'], record['token'])['parent'].parent) + free = filesystem.f_bavail * filesystem.f_frsize + require(available >= 10 * 1024**3 and free >= 48 * 1024**3, 'fresh host capacity gate refused') + return {'available_bytes': available, 'free_disk_bytes': free, 'os': version, 'build': build} + + +def observe_attachment(record): + scoped = paths(record['parent'], record['token']) + entries = plistlib.loads(call(record, ['/usr/bin/hdiutil', 'info', '-plist']))['images'] + matches = [entry for entry in entries if pathlib.Path(entry['image-path']).resolve() == scoped['image']] + require(len(matches) == 1, 'setup image attachment missing or ambiguous') + entities = matches[0]['system-entities'] + require(any(e.get('mount-point') == str(scoped['mount']) for e in entities), 'setup image mount missing') + devices = [] + for entity in entities: + device = entity.get('dev-entry', '') + if re.fullmatch('/dev/disk[0-9]+', device): + info = plistlib.loads(call(record, ['/usr/sbin/diskutil', 'info', '-plist', device])) + if info['TotalSize'] == STORE_BYTES:devices.append(device) + require(len(devices) == 1, 'fixed setup device is ambiguous') + volume = plistlib.loads(call(record, ['/usr/sbin/diskutil', 'info', '-plist', str(scoped['mount'])])) + require(volume['FilesystemType'] == 'apfs' and volume['MountPoint'] == str(scoped['mount']) + and 0 < volume['TotalSize'] <= STORE_BYTES + and re.fullmatch('[0-9A-Fa-f-]{36}', volume['VolumeUUID']), 'wrong setup volume readback') + return {'whole_device': devices[0], 'volume_uuid': volume['VolumeUUID']} + + +def remove_store(record): + scoped = paths(record['parent'], record['token']) + call(record, ['/usr/bin/hdiutil', 'detach', record['whole_device']]) + require(not os.path.ismount(scoped['mount']), 'owned host store remains mounted') + info = scoped['image'].lstat() + require((info.st_dev, info.st_ino) == (record['image_dev'], record['image_ino']) + and info.st_nlink == 1 and stat.S_ISREG(info.st_mode), 'host image changed before deletion') + scoped['image'].unlink() + + +def prepare(record, template): + """Resident watchdog owns setup before the first disk mutation. No VM start.""" + scoped = paths(record['parent'], record['token']) + proposal = plan(record['parent'], record['token'], record['limactl'], + str(scoped['mount'] / 'a/guest.raw.gz'), template) + observed = capacity(record) + for key, size, sha in (('limactl', 35_175_712, LIMA_SHA), ('guest_agent', 8_257_812, AGENT_SHA), + ('cached_image', IMAGE_BYTES, IMAGE_SHA)): + input_bytes(record[key], size, sha, deadline=record['_stage_mono']) + scoped['parent'].mkdir(mode=0o700) + record['_setup_tmp'] = str(scoped['parent']) + exclusive(scoped['parent'] / 'setup-ready.json', {'guardian_sha256': record['guardian_sha256'], + 'runner_commit': record['runner_commit'], 'manifest_sha256': record['manifest_sha256'], + 'process': process_identity(os.getpid()), 'capacity': observed}) + attached = False + try: + record['_command_seconds'] = 180 + call(record, proposal['create_store']) + info = scoped['image'].lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_uid == os.getuid() + and 0 < info.st_size <= STORE_BYTES + 2 * 1024**2, 'wrong newly created store') + record.update(image_dev=info.st_dev, image_ino=info.st_ino) + scoped['mount'].mkdir(mode=0o700) + call(record, proposal['attach_store']) + record.update(observe_attachment(record)) + attached = True + verify_store(record, profile_required=False) + for directory in ('a', 'l', 'tmp'):(scoped['mount'] / directory).mkdir(mode=0o700) + record.pop('_setup_tmp') + input_bytes(record['cached_image'], IMAGE_BYTES, IMAGE_SHA, scoped['mount'] / 'a/guest.raw.gz', record['_stage_mono']) + record['profile_sha256'] = exclusive(scoped['mount'] / 'guest.yaml', proposal['profile']) + require(record['profile_sha256'] == proposal['profile_sha256'], 'rendered setup template changed') + value = {key: record[key] for key in AUTHORITY_KEYS if key != 'schema'} + value['schema'] = 'durable-workflow.condition-host/v1' + record_sha = exclusive(scoped['parent'] / 'host-record.json', value) + verify_store(record) + require(time.monotonic() < record['_stage_mono'], 'setup window exhausted') + for key in ('_stage_mono', '_command_seconds'):record.pop(key, None) + return value, record_sha + except BaseException as error: + # Failed native create/attach may not return enough identity to authorize + # destructive recovery. Uncertain objects remain explicit external custody. + record.pop('_stage_mono', None) + record['_command_seconds'] = 20 + disposition = 'retained_uncertain_task_store' + if not attached and 'image_ino' in record: + try: + record.update(observe_attachment(record)) + attached = True + except BaseException: + pass + if attached: + try: + verify_store(record, profile_required=False) + remove_store(record) + disposition = 'owned_store_removed' + except BaseException: + pass + exclusive(scoped['parent'] / 'setup-failure.json', {'outcome': 'fail', + 'error_type': type(error).__name__, 'custody': disposition, + 'runner_commit': record['runner_commit'], 'manifest_sha256': record['manifest_sha256'], + 'store_authority': {key: record[key] for key in ('image_dev', 'image_ino', 'whole_device', 'volume_uuid') if key in record}}) + raise def capture_agent(record): @@ -303,13 +485,7 @@ def cleanup(record): call(record, [record['limactl'], '--tty=false', 'delete', '--force', scoped['name']]) require(not scoped['instance'].exists(), 'VM instance storage survived') verify_store(record) - call(record, ['/usr/bin/hdiutil', 'detach', record['whole_device']]) - require(not os.path.ismount(scoped['mount']), 'owned host store remains mounted') - # A changed inode never gains deletion authority through the pathname. - info = scoped['image'].lstat() - require((info.st_dev, info.st_ino) == (record['image_dev'], record['image_ino']) and info.st_nlink == 1, - 'host image changed before deletion') - scoped['image'].unlink() + remove_store(record) return {'guest_absent': True, 'store_unmounted': True, 'image_absent': True, 'image_cache_disposition': 'whole_task_guest_and_store_removed'} @@ -345,10 +521,11 @@ def audit(record, record_sha, source_sha): def main(argv=None): parser = argparse.ArgumentParser() - parser.add_argument('mode', choices=('plan', 'guard', 'audit')) + parser.add_argument('mode', choices=('plan', 'prepare', 'guard', 'audit')) parser.add_argument('--record') parser.add_argument('--record-sha256') parser.add_argument('--plan-input') + parser.add_argument('--profile') args = parser.parse_args(argv) if args.mode == 'plan': value = json.loads(pathlib.Path(args.plan_input).read_bytes()) @@ -356,9 +533,17 @@ def main(argv=None): return require('-I' in sys.orig_argv and '-S' in sys.orig_argv and '-c' in sys.orig_argv, 'captured host guardian required') source_sha = digest(sys.orig_argv[sys.orig_argv.index('-c') + 1].encode()) + mode = args.mode + if mode == 'prepare': + initial = setup_intent(pathlib.Path(args.record), args.record_sha256, source_sha) + template = json.loads(regular(pathlib.Path(args.profile), 64 * 1024)) + _, record_sha = prepare(initial, template) + args.record = str(paths(initial['parent'], initial['token'])['parent'] / 'host-record.json') + args.record_sha256 = record_sha + mode = 'guard' record = authority(pathlib.Path(args.record), args.record_sha256, source_sha) scoped = paths(record['parent'], record['token']) - if args.mode == 'guard': + if mode == 'guard': require(record['started_unix'] <= time.time() < record['started_unix'] + 300, 'host setup cutoff exceeded') verify_store(record) diff --git a/scripts/acceptance/condition-history/guest-proposal.md b/scripts/acceptance/condition-history/guest-proposal.md index 3a40c2d..2cb744a 100644 --- a/scripts/acceptance/condition-history/guest-proposal.md +++ b/scripts/acceptance/condition-history/guest-proposal.md @@ -65,36 +65,50 @@ and resource/deadline budgets. It accepts private canonical parent P named private path must satisfy Lima's Unix socket limit. No shell interpolation is used for native controls. -1. Before the first mutation, an external setup owner captures R/M, S and both +1. Before the first mutation, the external owner captures R/M, S and both cutoffs, input/tool hashes, architecture, OS, native Python version, free space - and current capacity. It creates an exclusive owner-only P and records intent. - An outside-guest setup watchdog owns the 300-second setup cutoff and partial - image/attachment custody. This source packet does not implement that setup - watchdog: its captured source and precise invocation are a remaining admission - prerequisite, even if every inert control passes. -2. Use only the plan's `hdiutil create -sectors 67108864 -type UDIF -layout + and current capacity. Write an exclusive 0600 setup intent outside the fresh P, + schema `durable-workflow.condition-host-setup/v1`, with exactly `INTENT_KEYS` + from the captured source. It binds the guardian SHA, R/M, host boot UUID, + token, S/cutoffs, scope, launcher/agent and already cached public image paths. + Capture its digest externally. Start the captured guardian in a detached new + session with native Python `-I -S -c`, mode `prepare`, `--record INTENT`, + `--record-sha256 INTENT_SHA` and `--profile CAPTURED_PROFILE`. No mutable + checkout executable is loaded. This resident process is the setup watchdog; + it owns the first native disk mutation and remains the guardian afterward. +2. The watchdog verifies the closed intent, pinned template and all three public + input hashes, native Intel architecture, exact OS and fresh RAM/disk gates. + Insufficient capacity refuses before even creating P. It creates P exclusively + at 0700 and writes `setup-ready.json` with its kernel epoch and capacity. + Only its plan's `hdiutil create -sectors 67108864 -type UDIF -layout GPTSPUD -fs APFS -nospotlight` and `attach -nobrowse -owners on -mountpoint P/v - -plist` commands. Capture image device/inode, the whole `/dev/diskN`, APFS volume + -plist` commands run. Native command output has the hard in-memory ceiling; + create/attach are each additionally bounded by the remaining 300-second setup + cutoff. Setup temporary files stay in P until the fixed filesystem is mounted, + then all VM/input/cache/tmp/log writes use that filesystem. Capture image + device/inode, the whole `/dev/diskN`, APFS volume UUID and the 32-GiB whole-device readback. Refuse an existing image, attachment, - mount, Lima home or instance. Copy the already verified public gzip and guest - agent into the store without networking, check hashes again, and write the + mount, Lima home or instance. Copy the already verified public gzip into the + store without networking, check its hash again, and write the rendered JSON-as-YAML profile exclusively. No private host source or credential - tree is copied. Local image input and disabled containerd downloads avoid - Lima's global downloader cache. -3. Write an exclusive 0600 authority record with schema + tree is copied. The installed public guest agent remains a pinned read-only + input. Local image input and disabled containerd downloads avoid Lima's global + downloader cache. Native readbacks must identify exactly one 32-GiB device, + one matching image attachment and an APFS volume at the exact mount point. +3. The resident watchdog writes an exclusive 0600 authority record with schema `durable-workflow.condition-host/v1`. Its exact fields are `AUTHORITY_KEYS` in `guest-host.py`: R/M, guardian digest, token, host boot UUID, S/cutoffs, private scope and launcher/agent paths, image device/inode, whole device, volume UUID - and rendered-profile digest. Capture its SHA-256 outside the store. The - external setup watchdog retains custody until the guardian is ready. -4. Start **captured Git bytes**, not a mutable script path, with native Python - `-I -S -c`, mode `guard`, `--record` and `--record-sha256`. The guardian verifies + and rendered-profile digest. Its SHA-256 is captured outside the store. +4. The same process verifies its new record and becomes the active guardian + without a process or custody gap. Standalone `guard` mode can only consume an + already admitted complete authority record. The guardian verifies launcher/agent digests, host boot, authority, image inode/geometry, attachment and profile. Its exclusive `host-ready.json` binds authority/source and the guardian's kernel process epoch. Only after independent readiness verification - may the setup owner run the exact scoped Lima `create` and `start` vectors. - The VM must not start before this custody transfer. Setup failure before that - transfer remains the setup watchdog's responsibility. + may the external owner run the exact scoped Lima `create` and `start` vectors. + The VM must not start before this readiness verification. The prepared mode + itself cannot create or start a VM. 5. Lima receives a fresh `LIMA_HOME`, a minimal environment, and no SSH agent, Docker context, proxy or cloud environment. Its host HOME remains the actual account home, with host SSH-key loading disabled. The complete profile has no @@ -135,8 +149,13 @@ used for native controls. digest agreement. Its exclusive `host-audit.json` may then record pass. A guest pass alone remains `pending_external_guest_teardown`. -Failure at any stage is terminal for this attempt. The external setup owner -handles partial create/attach failures under its setup watchdog. The guardian +Failure at any stage is terminal for this attempt. The resident setup watchdog +handles partial create/attach/copy failures within the unchanged cleanup cutoff. +After a lost attach response, it attempts a read-only recovery of the complete +attachment identity before authorizing detach. A failed create or uncertain +identity retains the task image explicitly; `setup-failure.json` records the +captured inode/device/volume authority it actually obtained. It never guesses +ownership or declares absence after an uncertain native result. The guardian attempts cleanup after malformed exports or failed process observation. If a device, inode, PID epoch or mount changed, it refuses destructive guesses and records retained custody for the external owner, who must resolve it before any @@ -146,10 +165,16 @@ recovery case, not an unattended-success claim. ## Remaining review gates -This packet intentionally cannot create/start a guest by itself. Before asking -for execution, supply the private setup/watchdog source and exact handoff, verify -the native APFS/Lima readback schema and guest dependency expectations, and obtain -fresh sufficient capacity. Security reviews the full executable/storage packet; +This packet intentionally cannot create/start a guest by itself. Setup watchdog +and teardown source are included in the same sealed guardian. Read-only native +inspection confirmed the supported `diskutil` property names and Darwin epoch +structure; fake native controls bind physical/synthesized devices, APFS geometry +and mount/UUID authority. No new attachment or guest readback is claimed. +Before execution, supply the exact private intent/command/handoff and fresh +sufficient capacity. The guest dependency assertions and all effective native +readbacks must pass on the sole admitted attempt; a missing tool or unexpected +schema fails rather than authorizing installation or a fallback. Security reviews +the full executable/storage packet; General admits its exact R/M/command/window. Any admitted source change requires new exact review. The old missing #191 raw capture and PHP-only #192 producer provenance remain explicit; this guest does not alter either acceptance boundary. diff --git a/scripts/acceptance/condition-history/test_guest_host.py b/scripts/acceptance/condition-history/test_guest_host.py index 7dd41ef..eac1679 100644 --- a/scripts/acceptance/condition-history/test_guest_host.py +++ b/scripts/acceptance/condition-history/test_guest_host.py @@ -4,10 +4,12 @@ import json import os import pathlib +import plistlib import shutil import sys import tempfile import time +import types import unittest from unittest import mock from test_subject import HERE, load @@ -174,6 +176,115 @@ def test_bad_guard_observation_still_attempts_cleanup(self): self.assertEqual(receipt['outcome'], 'fail') self.assertEqual(receipt['control_error_type'], 'SystemExit') + def fake_setup(self, failure=None): + self.parent.rmdir() + self.record['_stage_mono'] = time.monotonic() + 300 + self.record.update(schema='durable-workflow.condition-host-setup/v1', host_boot_id=BOOT, + guest_agent='/public/guest-agent', cached_image='/public/guest.raw.gz') + proposal = self.proposal() + calls = [] + def command(record, argv): + calls.append(argv) + if argv == proposal['create_store']: + if failure == 'create':raise SystemExit('failed native create') + self.scoped['image'].write_bytes(b'fixed-image') + if argv == proposal['attach_store'] and failure == 'attach':raise SystemExit('lost attach response') + return b'' + def copy_input(path, size, sha, destination=None, deadline=None): + if destination: + if failure == 'copy':raise OSError('fixed disk full') + destination.write_bytes(b'pinned-public-image') + return proposal, calls, command, copy_input + + def test_resident_watchdog_owns_setup_and_never_starts_guest(self): + proposal, calls, command, copy_input = self.fake_setup() + with mock.patch.object(host, 'capacity', return_value={'available_bytes': 12 * 1024**3}), \ + mock.patch.object(host, 'input_bytes', side_effect=copy_input), \ + mock.patch.object(host, 'call', side_effect=command), mock.patch.object(host, 'verify_store'), \ + mock.patch.object(host, 'process_identity', return_value={'pid': 1234}), \ + mock.patch.object(host, 'observe_attachment', return_value={'whole_device': '/dev/disk9', 'volume_uuid': BOOT}): + value, sha = host.prepare(self.record, self.profile) + self.assertEqual(set(value), host.AUTHORITY_KEYS) + self.assertEqual(sha, host.digest((self.parent / 'host-record.json').read_bytes())) + self.assertEqual(calls, [proposal['create_store'], proposal['attach_store']]) + self.assertTrue((self.parent / 'setup-ready.json').exists()) + self.assertEqual(value['profile_sha256'], proposal['profile_sha256']) + self.assertNotIn('_stage_mono', self.record) + + def test_setup_failures_have_owned_or_explicit_uncertain_custody(self): + for failure in ('create', 'attach', 'copy'): + with self.subTest(stage=failure): + if self.parent.exists():shutil.rmtree(self.parent) + self.parent.mkdir(mode=0o700) + _, _, command, copy_input = self.fake_setup(failure) + with mock.patch.object(host, 'capacity', return_value={}), \ + mock.patch.object(host, 'input_bytes', side_effect=copy_input), \ + mock.patch.object(host, 'call', side_effect=command), mock.patch.object(host, 'verify_store'), \ + mock.patch.object(host, 'process_identity', return_value={'pid': 1234}), \ + mock.patch.object(host, 'observe_attachment', return_value={'whole_device': '/dev/disk9', 'volume_uuid': BOOT}), \ + mock.patch.object(host, 'remove_store') as remove, self.assertRaises((SystemExit, OSError)): + host.prepare(self.record, self.profile) + receipt = json.loads((self.parent / 'setup-failure.json').read_bytes()) + self.assertEqual(receipt['outcome'], 'fail') + if failure == 'create': + self.assertEqual(receipt['custody'], 'retained_uncertain_task_store') + remove.assert_not_called() + else: + self.assertEqual(receipt['custody'], 'owned_store_removed') + remove.assert_called_once() + self.assertIn('image_ino', receipt['store_authority']) + self.record.pop('image_ino', None) + self.record.pop('image_dev', None) + + def test_capacity_refuses_before_creating_a_disk_or_scope(self): + self.parent.rmdir() + with mock.patch.object(host, 'capacity', side_effect=SystemExit('insufficient memory')), \ + mock.patch.object(host, 'call', side_effect=AssertionError('no native mutation')), self.assertRaises(SystemExit): + host.prepare(self.record, self.profile) + self.assertFalse(self.parent.exists()) + + def test_native_memory_and_disk_readbacks_have_exact_refusal_gates(self): + def native(record, argv): + if '-productVersion' in argv:return b'15.8.2\n' + if '-buildVersion' in argv:return b'24H232\n' + return b'Mach Virtual Memory Statistics: (page size of 4096 bytes)\nPages free: 2000000.\nPages inactive: 1000000.\nPages speculative: 0.\n' + with mock.patch.object(host.sys, 'platform', 'darwin'), \ + mock.patch.object(host.os, 'uname', return_value=types.SimpleNamespace(machine='x86_64')), \ + mock.patch.object(host, 'call', side_effect=native), \ + mock.patch.object(host.os, 'statvfs', return_value=types.SimpleNamespace(f_bavail=60 * 1024**3, f_frsize=1)): + self.assertEqual(host.capacity(self.record)['available_bytes'], 3_000_000 * 4096) + with mock.patch.object(host.os, 'statvfs', return_value=types.SimpleNamespace(f_bavail=1024, f_frsize=1)), \ + self.assertRaises(SystemExit):host.capacity(self.record) + with mock.patch.object(host.sys, 'platform', 'darwin'), \ + mock.patch.object(host.os, 'uname', return_value=types.SimpleNamespace(machine='x86_64')), \ + mock.patch.object(host, 'call', return_value=b'unknown'), self.assertRaises(SystemExit): + host.capacity(self.record) + + def test_pinned_input_copy_refuses_changed_bytes_existing_destination_and_cutoff(self): + path = self.parent / 'public';path.write_bytes(b'public immutable input') + sha = host.digest(path.read_bytes());size = path.stat().st_size + target = self.parent / 'copied' + host.input_bytes(path, size, sha, target, time.monotonic() + 2) + self.assertEqual(path.read_bytes(), target.read_bytes()) + with self.assertRaises(FileExistsError):host.input_bytes(path, size, sha, target) + for length, digest, deadline in ((size + 1, sha, None), (size, 'f' * 64, None), (size, sha, time.monotonic() - 1)): + with self.subTest(length=length, digest=digest), self.assertRaises(SystemExit): + host.input_bytes(path, length, digest, deadline=deadline) + + def test_native_attachment_requires_unique_fixed_device_and_exact_mount(self): + attachment = {'images': [{'image-path': str(self.scoped['image']), 'system-entities': [ + {'dev-entry': '/dev/disk9'}, {'dev-entry': '/dev/disk10', 'mount-point': str(self.scoped['mount'])}]}]} + def native(record, argv): + if 'hdiutil' in argv[0]:return plistlib.dumps(attachment) + if argv[-1] == '/dev/disk9':return plistlib.dumps({'TotalSize': host.STORE_BYTES}) + if argv[-1] == '/dev/disk10':return plistlib.dumps({'TotalSize': host.STORE_BYTES - 4096}) + return plistlib.dumps({'FilesystemType': 'apfs', 'MountPoint': str(self.scoped['mount']), + 'TotalSize': host.STORE_BYTES - 4096, 'VolumeUUID': BOOT}) + with mock.patch.object(host, 'call', side_effect=native): + self.assertEqual(host.observe_attachment(self.record), {'whole_device': '/dev/disk9', 'volume_uuid': BOOT}) + attachment['images'][0]['system-entities'][1]['mount-point'] = '/foreign' + with self.assertRaises(SystemExit):host.observe_attachment(self.record) + if __name__ == '__main__': unittest.main() From 0736fc85c60a7f75335e06590abf762df7903620 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 03:27:33 +0000 Subject: [PATCH 10/14] fix: bind guest image profile and agent consumption to the guardian --- .../acceptance/condition-history/README.md | 6 +- .../condition-history/guest-host.py | 192 +++++++++++++++++- .../condition-history/guest-proposal.md | 55 ++++- .../condition-history/test_guest_host.py | 163 ++++++++++++++- 4 files changed, 393 insertions(+), 23 deletions(-) diff --git a/scripts/acceptance/condition-history/README.md b/scripts/acceptance/condition-history/README.md index 507d561..2fe3df7 100644 --- a/scripts/acceptance/condition-history/README.md +++ b/scripts/acceptance/condition-history/README.md @@ -182,7 +182,11 @@ guesses which layers to delete from a shared daemon. The concrete [guest/guardian source proposal](guest-proposal.md) selects cached native Lima/VZ inputs and a fixed enclosing host disk. The captured external guardian also owns bounded setup before allocation. Exact source review, private handoff and fresh -capacity remain prerequisites; no VM execution is admitted. +capacity remain prerequisites; no VM execution is admitted. After an exact +private request, that guardian owns create from captured stdin, rechecks copied +image identity, records effective configuration, starts the guest and verifies +the running guest-agent executable before any consumer. These are source +assertions under review, not observed guest results. The comparator scans every opened wait, condition outcome and timer row, binding the authored group, member key/fingerprint/occurrence, wait ID and integer timeout. diff --git a/scripts/acceptance/condition-history/guest-host.py b/scripts/acceptance/condition-history/guest-host.py index 02386b6..cdd4df6 100644 --- a/scripts/acceptance/condition-history/guest-host.py +++ b/scripts/acceptance/condition-history/guest-host.py @@ -1,11 +1,12 @@ """Source proposal for an external, task-scoped Lima/VZ storage guardian. -The plan is inert. Prepare allocates native storage only after separate admission; -guard/audit consume captured private authority. No mode creates or starts a VM. +The plan is inert. Prepare/guard may allocate storage and consume an exact +private guest-launch request only after separate admission; audit is read-only. """ import argparse import ctypes import errno +import gzip import hashlib import json import os @@ -21,6 +22,7 @@ LIMA_SHA = '216450fc8e3eeb7f50e3030de216c32efeef505b111ed3fb24912a36feebea49' AGENT_SHA = '4aa75eacc151cbf2b7aec1b9ebe3886a0fc3268977c108308c802ce267c76959' +AGENT_BINARY_SHA = 'd799765623d5fa3b52ac30dd437b8f867968aeac594933c8ddc3bf1ac78cb26a' IMAGE_SHA = '4cd967d2c58aa6971621343d372255b14ffbfe28a185cda0e3c34ac8a1eb15e6' IMAGE_BYTES = 358_298_593 STORE_BYTES = 32 * 1024**3 @@ -30,8 +32,9 @@ AUTHORITY_KEYS = {'schema', 'token', 'runner_commit', 'manifest_sha256', 'guardian_sha256', 'started_unix', 'active_unix', 'cleanup_unix', 'parent', 'limactl', 'guest_agent', 'host_boot_id', 'image_dev', 'image_ino', 'whole_device', - 'volume_uuid', 'profile_sha256'} -INTENT_KEYS = (AUTHORITY_KEYS - {'image_dev', 'image_ino', 'whole_device', 'volume_uuid', 'profile_sha256'}) | {'cached_image'} + 'volume_uuid', 'profile_sha256', 'profile', 'guest_image', 'guest_agent_binary_sha256'} +INTENT_KEYS = (AUTHORITY_KEYS - {'image_dev', 'image_ino', 'whole_device', 'volume_uuid', + 'profile_sha256', 'profile', 'guest_image', 'guest_agent_binary_sha256'}) | {'cached_image'} def require(ok, label): @@ -122,22 +125,27 @@ def plan(parent, token, limactl, image_path, profile): '-volname', 'sr-' + token, str(scoped['image'])], 'attach_store': ['/usr/bin/hdiutil', 'attach', '-nobrowse', '-owners', 'on', '-mountpoint', str(scoped['mount']), '-plist', str(scoped['image'])], - 'create_vm': [limactl, '--tty=false', 'create', '--name', scoped['name'], str(scoped['mount'] / 'guest.yaml')], + 'create_vm': [limactl, '--tty=false', 'create', '--name', scoped['name'], '-'], 'start_vm': [limactl, '--tty=false', 'start', '--timeout', '5m', scoped['name']], 'stop_vm': [limactl, '--tty=false', 'stop', '--force', scoped['name']], 'delete_vm': [limactl, '--tty=false', 'delete', '--force', scoped['name']], 'environment': environment({'parent': parent, 'token': token})} -def call(record, argv): +def call(record, argv, input_data=None): remaining = min(record['_cleanup_mono'], record.get('_stage_mono', float('inf'))) - time.monotonic() require(remaining > 0, 'external cleanup deadline exhausted') # Read with a hard in-memory ceiling; native commands cannot fill host logs. - child = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + require(input_data is None or len(input_data) <= 8192, 'oversize native input') + child = subprocess.Popen(argv, stdin=subprocess.PIPE if input_data is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=environment(record), close_fds=True) data = bytearray() cutoff = time.monotonic() + min(record.get('_command_seconds', 20), remaining) try: + if input_data is not None: + child.stdin.write(input_data) + child.stdin.close() with selectors.DefaultSelector() as selector: selector.register(child.stdout, selectors.EVENT_READ) while selector.get_map(): @@ -215,6 +223,13 @@ def authority(path, expected_sha, source_sha): require(digest(binary.read_bytes()) == LIMA_SHA, 'reviewed VM launcher changed') agent = pathlib.Path(value['guest_agent']).resolve(strict=True) require(digest(agent.read_bytes()) == AGENT_SHA, 'reviewed guest agent changed') + selected_agent(value) + template = json.loads(json.dumps(value['profile'])) + template['images'][0]['location'] = '__TASK_IMAGE__' + expected_profile = plan(value['parent'], value['token'], value['limactl'], + str(scoped['mount'] / 'a/guest.raw.gz'), template)['profile'] + require(value['profile'] == expected_profile and digest(canonical(value['profile'])) == value['profile_sha256'] + and value['guest_agent_binary_sha256'] == AGENT_BINARY_SHA, 'captured profile or agent authority mismatch') require(re.fullmatch('/dev/disk[0-9]+', value['whole_device']) and re.fullmatch('[0-9A-Fa-f-]{36}', value['volume_uuid']) and re.fullmatch('[0-9a-f]{64}', value['profile_sha256']), 'invalid native store authority') @@ -284,6 +299,65 @@ def input_bytes(path, expected_size, expected_sha, destination=None, deadline=No if target:target.close() +def bound_input(path, expected_size, expected_sha, deadline=None): + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_uid == os.getuid(), + 'unsafe consumed guest input') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + state, count = hashlib.sha256(), 0 + with os.fdopen(fd, 'rb') as stream: + opened = os.fstat(stream.fileno()) + require((opened.st_dev, opened.st_ino) == (before.st_dev, before.st_ino), 'consumed guest input replaced on open') + while True: + if deadline is not None:require(time.monotonic() < deadline, 'consumed input cutoff') + block = stream.read(1024**2) + if not block:break + count += len(block) + require(count <= expected_size, 'consumed guest input grew') + state.update(block) + after = os.fstat(stream.fileno()) + current = path.lstat() + identity = lambda info: (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns, info.st_nlink) + require(identity(before) == identity(opened) == identity(after) == identity(current), 'consumed guest input changed') + require(count == expected_size and state.hexdigest() == expected_sha, 'consumed guest input digest mismatch') + return {'device': opened.st_dev, 'inode': opened.st_ino, 'bytes': expected_size, 'sha256': expected_sha} + + +def selected_agent(record): + # Lima 2.2.1 usrlocal.ShareLima/GuestAgentBinary uses this executable-relative + # compressed path first; instance.Start passes it as --guestagent to hostagent. + launcher = pathlib.Path(record['limactl']).resolve() + expected = launcher.parent.parent / 'share/lima/lima-guestagent.Linux-x86_64.gz' + require(pathlib.Path(record['limactl']) == launcher and pathlib.Path(record['guest_agent']) == expected + and expected.resolve() == expected, 'pinned agent is not the executable-selected Lima agent') + return expected + + +def agent_binary_sha(path, deadline=None): + state, count = hashlib.sha256(), 0 + with gzip.open(path, 'rb') as stream: + while True: + if deadline is not None:require(time.monotonic() < deadline, 'guest agent input cutoff') + data = stream.read(1024**2) + if not data:break + count += len(data) + require(count <= 64 * 1024**2, 'oversize uncompressed guest agent') + state.update(data) + require(count > 0, 'empty guest agent') + require(state.hexdigest() == AGENT_BINARY_SHA, 'uncompressed pinned guest agent changed') + input_bytes(path, 8_257_812, AGENT_SHA, deadline=deadline) + return state.hexdigest() + + +def verify_guest_inputs(record): + scoped = paths(record['parent'], record['token']) + identity = bound_input(scoped['mount'] / 'a/guest.raw.gz', IMAGE_BYTES, IMAGE_SHA, record['_active_mono']) + require(identity == record['guest_image'], 'consumed image identity differs from copied destination') + selected_agent(record) + input_bytes(record['guest_agent'], 8_257_812, AGENT_SHA, deadline=record['_active_mono']) + require(agent_binary_sha(record['guest_agent'], record['_active_mono']) == record['guest_agent_binary_sha256'], 'consumed guest agent changed') + + def setup_intent(path, expected_sha, source_sha): require(stat.S_IMODE(path.lstat().st_mode) == 0o600, 'private setup intent required') raw = regular(path, 64 * 1024) @@ -365,6 +439,7 @@ def prepare(record, template): proposal = plan(record['parent'], record['token'], record['limactl'], str(scoped['mount'] / 'a/guest.raw.gz'), template) observed = capacity(record) + selected_agent(record) for key, size, sha in (('limactl', 35_175_712, LIMA_SHA), ('guest_agent', 8_257_812, AGENT_SHA), ('cached_image', IMAGE_BYTES, IMAGE_SHA)): input_bytes(record[key], size, sha, deadline=record['_stage_mono']) @@ -389,6 +464,9 @@ def prepare(record, template): for directory in ('a', 'l', 'tmp'):(scoped['mount'] / directory).mkdir(mode=0o700) record.pop('_setup_tmp') input_bytes(record['cached_image'], IMAGE_BYTES, IMAGE_SHA, scoped['mount'] / 'a/guest.raw.gz', record['_stage_mono']) + record['guest_image'] = bound_input(scoped['mount'] / 'a/guest.raw.gz', IMAGE_BYTES, IMAGE_SHA, record['_stage_mono']) + record['guest_agent_binary_sha256'] = agent_binary_sha(record['guest_agent'], record['_stage_mono']) + record['profile'] = proposal['profile'] record['profile_sha256'] = exclusive(scoped['mount'] / 'guest.yaml', proposal['profile']) require(record['profile_sha256'] == proposal['profile_sha256'], 'rendered setup template changed') value = {key: record[key] for key in AUTHORITY_KEYS if key != 'schema'} @@ -439,6 +517,7 @@ def capture_agent(record): pairs = list(zip(argv, argv[1:])) require(argv and pathlib.Path(argv[0]).resolve() == pathlib.Path(record['limactl']).resolve() and 'hostagent' in argv and (binding in argv or ('--pidfile', str(pid_path)) in pairs) + and ('--guestagent', str(selected_agent(record))) in pairs and scoped['name'] in argv and process_identity(process['pid']) == process, 'foreign VM host agent') return process @@ -467,6 +546,97 @@ def finish_requested(record, record_sha): return True +def effective_config(record, status): + scoped = paths(record['parent'], record['token']) + value = json.loads(call(record, [record['limactl'], 'list', '--json', '--all-fields', scoped['name']])) + require(value['name'] == scoped['name'] and value['dir'] == str(scoped['instance']) + and value['status'] == status and not value.get('errors') + and value['vmType'] == 'vz' and value['arch'] == 'x86_64' and value['cpus'] == 4 + and value['memory'] == MEMORY_BYTES and value['disk'] == GUEST_BYTES, + 'effective VM identity or resources changed') + def contains(actual, expected): + if expected == [] or expected == {}: + return actual is None or actual == expected + if isinstance(expected, dict): + return isinstance(actual, dict) and all(k in actual and contains(actual[k], v) for k, v in expected.items()) + if isinstance(expected, list): + return isinstance(actual, list) and len(actual) == len(expected) and all(contains(a, e) for a, e in zip(actual, expected)) + return type(actual) is type(expected) and actual == expected + # base/minimum version are template loader directives; every effective + # supplied field is checked, including mounts, images, ssh and provisioning. + expected = {k: v for k, v in record['profile'].items() if k not in ('base', 'minimumLimaVersion')} + config = dict(value['config']) + config.setdefault('env', {}) + require(contains(config, expected), 'effective VM profile differs from captured configuration') + return value + + +GUEST_AGENT_PROBE = '''import hashlib,json,pathlib +root=pathlib.Path('/proc');boot=(root/'sys/kernel/random/boot_id').read_text().strip();found=[] +for entry in root.iterdir(): + if not entry.name.isdigit():continue + try:comm=(entry/'comm').read_text().strip() + except FileNotFoundError:continue + if comm!='lima-guestagent':continue + try: + first=(entry/'stat').read_text();state=hashlib.sha256();count=0 + with (entry/'exe').open('rb') as stream: + while True: + data=stream.read(1048576) + if not data:break + count+=len(data) + if count>67108864:raise SystemExit('oversize running guest agent') + state.update(data) + second=(entry/'stat').read_text() + start=lambda value:int(value[value.rfind(') ')+2:].split()[19]) + if start(first)!=start(second):raise SystemExit('guest agent epoch changed') + found.append({'pid':int(entry.name),'start_ticks':start(second),'guest_boot_id':boot,'sha256':state.hexdigest()}) + except FileNotFoundError:raise SystemExit('guest process changed during probe') +if len(found)!=1 or (root/'sys/kernel/random/boot_id').read_text().strip()!=boot:raise SystemExit('ambiguous guest agent authority') +print(json.dumps(found[0],sort_keys=True)) +''' + + +def launch_guest(record, record_sha): + scoped = paths(record['parent'], record['token']) + request = json.loads(regular(scoped['parent'] / 'launch.json', 64 * 1024)) + require(request == {'record_sha256': record_sha, 'manifest_sha256': record['manifest_sha256'], 'action': 'create-and-start'}, + 'wrong exact guest launch request') + require(not scoped['instance'].exists(), 'guest instance already exists') + verify_store(record) + verify_guest_inputs(record) + # The bytes consumed by create come from captured authority, never the + # mutable guest.yaml pathname. No external process owns create/start. + profile = canonical(record['profile']) + require(digest(profile) == record['profile_sha256'], 'captured consumption profile changed') + record['_stage_mono'] = record['_active_mono'] + record['_command_seconds'] = 180 + try: + call(record, [record['limactl'], '--tty=false', 'create', '--name', scoped['name'], '-'], profile) + before = effective_config(record, 'Stopped') + exclusive(scoped['parent'] / 'vm-created.json', {'record_sha256': record_sha, + 'profile_sha256': digest(profile), 'guest_image': record['guest_image'], 'effective': before}) + verify_guest_inputs(record) + require(effective_config(record, 'Stopped') == before, 'VM config changed before guardian start') + record['_command_seconds'] = 300 + call(record, [record['limactl'], '--tty=false', 'start', '--timeout', '5m', scoped['name']]) + after = effective_config(record, 'Running') + process = capture_agent(record) + require(process, 'missing consumed VM hostagent authority') + proof = json.loads(call(record, [record['limactl'], 'shell', scoped['name'], '--', + 'sudo', '-n', 'python3', '-I', '-S', '-c', GUEST_AGENT_PROBE])) + require(set(proof) == {'pid', 'start_ticks', 'guest_boot_id', 'sha256'} + and type(proof['pid']) is int and proof['pid'] > 1 + and type(proof['start_ticks']) is int and proof['start_ticks'] > 0 + and proof['sha256'] == record['guest_agent_binary_sha256'] + and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', proof['guest_boot_id']), + 'running guest agent does not match selected pinned agent') + exclusive(scoped['parent'] / 'vm-started.json', {'record_sha256': record_sha, + 'effective': after, 'host_process': process, 'agent': proof}) + finally: + for key in ('_stage_mono', '_command_seconds'):record.pop(key, None) + + def cleanup(record): scoped = paths(record['parent'], record['token']) verify_store(record) @@ -508,11 +678,15 @@ def audit(record, record_sha, source_sha): require(finish_requested(record, record_sha), 'completed guest export request required') guest = json.loads(regular(scoped['parent'] / 'guest-summary.json')) guest_audit = json.loads(regular(scoped['parent'] / 'guest-audit.json')) + started = json.loads(regular(scoped['parent'] / 'vm-started.json')) require(guest['runner_commit'] == record['runner_commit'] and guest['manifest_sha256'] == record['manifest_sha256'] and guest['checks_outcome'] == 'pass' and guest_audit['outcome'] == 'pending_external_guest_teardown' and guest['storage']['guest_boot_id'] == exports['guest_boot_id'] and guest_audit['storage'] == guest['storage'] and guest_audit['manifest_sha256'] == record['manifest_sha256'] + and started['record_sha256'] == record_sha + and started['agent']['guest_boot_id'] == exports['guest_boot_id'] + and started['agent']['sha256'] == record['guest_agent_binary_sha256'] and guest_audit['summary_sha256'] == exports['guest-summary.json'], 'guest evidence does not authorize acceptance') result.update(outcome='pass', exports_sha256=export_sha) exclusive(scoped['parent'] / 'host-audit.json', result) @@ -550,8 +724,12 @@ def main(argv=None): exclusive(scoped['parent'] / 'host-ready.json', {'record_sha256': args.record_sha256, 'source_sha256': source_sha, 'process': process_identity(os.getpid())}) control_error = None + launched = False try: while time.monotonic() < record['_active_mono']: + if not launched and os.path.lexists(scoped['parent'] / 'launch.json'): + launch_guest(record, args.record_sha256) + launched = True process = capture_agent(record) receipt = scoped['parent'] / 'vm-process.json' if process and not receipt.exists(): diff --git a/scripts/acceptance/condition-history/guest-proposal.md b/scripts/acceptance/condition-history/guest-proposal.md index 2cb744a..3f08525 100644 --- a/scripts/acceptance/condition-history/guest-proposal.md +++ b/scripts/acceptance/condition-history/guest-proposal.md @@ -20,6 +20,7 @@ execution proposal rather than this repository. | --- | --- | | `limactl` | Lima 2.2.1, 35,175,712 bytes, SHA-256 `216450fc8e3eeb7f50e3030de216c32efeef505b111ed3fb24912a36feebea49` | | Linux amd64 guest agent gzip | 8,257,812 bytes, SHA-256 `4aa75eacc151cbf2b7aec1b9ebe3886a0fc3268977c108308c802ce267c76959` | +| Uncompressed guest agent | SHA-256 `d799765623d5fa3b52ac30dd437b8f867968aeac594933c8ddc3bf1ac78cb26a` | | Cached guest image | [colima-core v0.10.4 Ubuntu 24.04 Docker amd64 raw gzip](https://github.com/abiosoft/colima-core/releases/download/v0.10.4/ubuntu-24.04-minimal-cloudimg-amd64-docker.raw.gz), 358,298,593 bytes, SHA-256 `4cd967d2c58aa6971621343d372255b14ffbfe28a185cda0e3c34ac8a1eb15e6` | | Native tools | macOS 15.8.2 build 24H232 `hdiutil`, `diskutil`, `sysctl`, public `libproc` process information | | Host guardian | Captured `guest-host.py` Git blob at admitted R, SHA-256 in the R manifest and private authority | @@ -89,7 +90,9 @@ used for native controls. device/inode, the whole `/dev/diskN`, APFS volume UUID and the 32-GiB whole-device readback. Refuse an existing image, attachment, mount, Lima home or instance. Copy the already verified public gzip into the - store without networking, check its hash again, and write the + store without networking, reopen/hash the destination through a no-follow + single-link owner file descriptor, and bind its device/inode/size/SHA. Recheck + destination identity and bytes at consumption. Write the rendered JSON-as-YAML profile exclusively. No private host source or credential tree is copied. The installed public guest agent remains a pinned read-only input. Local image input and disabled containerd downloads avoid Lima's global @@ -99,16 +102,24 @@ used for native controls. `durable-workflow.condition-host/v1`. Its exact fields are `AUTHORITY_KEYS` in `guest-host.py`: R/M, guardian digest, token, host boot UUID, S/cutoffs, private scope and launcher/agent paths, image device/inode, whole device, volume UUID - and rendered-profile digest. Its SHA-256 is captured outside the store. + rendered profile/digest, copied image identity and uncompressed guest-agent + digest. Its SHA-256 is captured outside the store. 4. The same process verifies its new record and becomes the active guardian without a process or custody gap. Standalone `guard` mode can only consume an already admitted complete authority record. The guardian verifies launcher/agent digests, host boot, authority, image inode/geometry, attachment and profile. Its exclusive `host-ready.json` binds authority/source and the guardian's kernel process epoch. Only after independent readiness verification - may the external owner run the exact scoped Lima `create` and `start` vectors. - The VM must not start before this readiness verification. The prepared mode - itself cannot create or start a VM. + may the external owner write an exclusive private `launch.json` with exactly + `record_sha256`, `manifest_sha256` and `action: "create-and-start"`. This request + is part of the separately admitted invocation. The resident guardian owns + both create and start. It revalidates the copied image and selected agent, + then passes the captured profile bytes on stdin to `limactl create --name NAME + -`; no mutable YAML pathname supplies executable settings. It records and + checks `limactl list --json --all-fields NAME` before start, including image, + resource, network, mount, SSH, forwarding, environment and provision settings. + Immediately before its own start call it repeats the input/config checks. + Setup/template file edits cannot replace the captured create input. 5. Lima receives a fresh `LIMA_HOME`, a minimal environment, and no SSH agent, Docker context, proxy or cloud environment. Its host HOME remains the actual account home, with host SSH-key loading disabled. The complete profile has no @@ -124,8 +135,24 @@ used for native controls. `limactl stop --force`. The guardian never sends a numeric host kill signal to a recovered PID. Missing, changed or foreign authority is a failed result requiring the external owner, not permission to stop another instance. -7. Obtain the guest's Linux boot ID independently over the generated SSH control - path. Transfer a clean public R checkout with the immutable H tag and invoke + The supported Lima 2.2.1 selection is causal: the canonical launcher resolves + `/share/lima/lima-guestagent.Linux-x86_64.gz` through + `usrlocal.GuestAgentBinary`, and `instance.Start` passes it as `--guestagent` + to hostagent. The source refuses another correctly hashed path, and the + independent hostagent command readback must name this exact selected path. + Primary source: [agent selection](https://github.com/lima-vm/lima/blob/v2.2.1/pkg/usrlocal/usrlocal.go) + and [start argument construction](https://github.com/lima-vm/lima/blob/v2.2.1/pkg/instance/start.go). +7. After start, the guardian repeats effective configuration readback and runs + a captured read-only probe over Lima's generated SSH control connection. In + the disposable guest only, passwordless sudo lets the probe hash the unique + running guest agent's `/proc/PID/exe`, with consistent Linux boot/start ticks. + That kernel executable hash must match the pinned uncompressed agent above. + A path declaration alone cannot authorize the receipt. Exclusive + `vm-started.json` binds the actual guest boot, agent hash, hostagent epoch and + effective configuration; no SDK consumer is released before this receipt. + These probes are source under review and have not run in a guest. + Independently obtain B over the generated control path and match that receipt. + Transfer a clean public R checkout with the immutable H tag and invoke the README's captured entry with that B and the same S/cutoffs. No host source mount or credential is used. The guest requires an empty daemon/build cache and ext4 root/work/Docker data on its fixed block device before pulling. @@ -165,11 +192,17 @@ recovery case, not an unattended-success claim. ## Remaining review gates -This packet intentionally cannot create/start a guest by itself. Setup watchdog -and teardown source are included in the same sealed guardian. Read-only native +The packet requires both separate execution admission and the exact private +launch request before it can create/start a guest. Setup watchdog, input +consumption and teardown source are included in the same sealed guardian. Read-only native inspection confirmed the supported `diskutil` property names and Darwin epoch -structure; fake native controls bind physical/synthesized devices, APFS geometry -and mount/UUID authority. No new attachment or guest readback is claimed. +structure. Native CLI help also confirms stdin create and effective Config +readback support; a read-only public-agent probe confirms executable-relative +selection and its uncompressed digest. Fake native controls bind +physical/synthesized devices, APFS geometry and mount/UUID authority. Actual held +source countercontrols expose destination corruption, post-ready YAML +edit/restore and an unrelated hashed agent path; candidate controls refuse them. +No new attachment, VM or guest-kernel readback is claimed. Before execution, supply the exact private intent/command/handoff and fresh sufficient capacity. The guest dependency assertions and all effective native readbacks must pass on the sole admitted attempt; a missing tool or unexpected diff --git a/scripts/acceptance/condition-history/test_guest_host.py b/scripts/acceptance/condition-history/test_guest_host.py index eac1679..ed2e3e7 100644 --- a/scripts/acceptance/condition-history/test_guest_host.py +++ b/scripts/acceptance/condition-history/test_guest_host.py @@ -6,6 +6,7 @@ import pathlib import plistlib import shutil +import subprocess import sys import tempfile import time @@ -25,10 +26,12 @@ def setUp(self): self.addCleanup(shutil.rmtree, self.temp) self.parent = self.temp / ('sr-' + TOKEN) self.parent.mkdir(mode=0o700) - self.record = {'parent': str(self.parent), 'token': TOKEN, 'limactl': '/reviewed/limactl', + self.record = {'parent': str(self.parent), 'token': TOKEN, 'limactl': '/reviewed/bin/limactl', 'runner_commit': 'a' * 40, 'manifest_sha256': 'b' * 64, 'guardian_sha256': 'c' * 64, + 'guest_agent_binary_sha256': 'e' * 64, 'started_unix': int(time.time()), 'active_unix': int(time.time()) + 1740, - 'cleanup_unix': int(time.time()) + 1800, '_cleanup_mono': time.monotonic() + 1800} + 'cleanup_unix': int(time.time()) + 1800, '_cleanup_mono': time.monotonic() + 1800, + '_active_mono': time.monotonic() + 1740} self.scoped = host.paths(str(self.parent), TOKEN) self.profile = json.loads((HERE / 'guest-profile.json').read_bytes()) @@ -110,6 +113,8 @@ def audit_files(self): self.put('host-ready.json', {'record_sha256': 'd' * 64, 'source_sha256': 'c' * 64, 'process': {'pid': 1234}}) self.put('vm-process.json', {'pid': 2345}) + self.put('vm-started.json', {'record_sha256': 'd' * 64, + 'agent': {'guest_boot_id': BOOT, 'sha256': self.record['guest_agent_binary_sha256']}}) self.put('host-cleanup.json', {'record_sha256': 'd' * 64, 'guardian_sha256': 'c' * 64, 'outcome': 'pending_independent_host_audit', 'guest_absent': True, 'store_unmounted': True, 'image_absent': True}) @@ -131,7 +136,8 @@ def observe(pid):return {'pid': pid} if pid == survivor else None def test_host_audit_refuses_rebound_guest_and_forged_cleanup(self): self.audit_files() for filename, key, value in [('guest-summary.json', 'runner_commit', 'e' * 40), - ('guest-audit.json', 'outcome', 'pass'), ('host-cleanup.json', 'guardian_sha256', 'foreign')]: + ('guest-audit.json', 'outcome', 'pass'), ('host-cleanup.json', 'guardian_sha256', 'foreign'), + ('vm-started.json', 'agent', {'guest_boot_id': BOOT, 'sha256': 'foreign'})]: self.audit_files() path = self.parent / filename data = json.loads(path.read_bytes());data[key] = value @@ -180,7 +186,7 @@ def fake_setup(self, failure=None): self.parent.rmdir() self.record['_stage_mono'] = time.monotonic() + 300 self.record.update(schema='durable-workflow.condition-host-setup/v1', host_boot_id=BOOT, - guest_agent='/public/guest-agent', cached_image='/public/guest.raw.gz') + guest_agent='/reviewed/share/lima/lima-guestagent.Linux-x86_64.gz', cached_image='/public/guest.raw.gz') proposal = self.proposal() calls = [] def command(record, argv): @@ -200,6 +206,8 @@ def test_resident_watchdog_owns_setup_and_never_starts_guest(self): proposal, calls, command, copy_input = self.fake_setup() with mock.patch.object(host, 'capacity', return_value={'available_bytes': 12 * 1024**3}), \ mock.patch.object(host, 'input_bytes', side_effect=copy_input), \ + mock.patch.object(host, 'bound_input', return_value={'device': 1, 'inode': 2, 'bytes': host.IMAGE_BYTES, 'sha256': host.IMAGE_SHA}), \ + mock.patch.object(host, 'agent_binary_sha', return_value='e' * 64), \ mock.patch.object(host, 'call', side_effect=command), mock.patch.object(host, 'verify_store'), \ mock.patch.object(host, 'process_identity', return_value={'pid': 1234}), \ mock.patch.object(host, 'observe_attachment', return_value={'whole_device': '/dev/disk9', 'volume_uuid': BOOT}): @@ -219,6 +227,8 @@ def test_setup_failures_have_owned_or_explicit_uncertain_custody(self): _, _, command, copy_input = self.fake_setup(failure) with mock.patch.object(host, 'capacity', return_value={}), \ mock.patch.object(host, 'input_bytes', side_effect=copy_input), \ + mock.patch.object(host, 'bound_input', return_value={'device': 1, 'inode': 2, 'bytes': host.IMAGE_BYTES, 'sha256': host.IMAGE_SHA}), \ + mock.patch.object(host, 'agent_binary_sha', return_value='e' * 64), \ mock.patch.object(host, 'call', side_effect=command), mock.patch.object(host, 'verify_store'), \ mock.patch.object(host, 'process_identity', return_value={'pid': 1234}), \ mock.patch.object(host, 'observe_attachment', return_value={'whole_device': '/dev/disk9', 'volume_uuid': BOOT}), \ @@ -285,6 +295,151 @@ def native(record, argv): attachment['images'][0]['system-entities'][1]['mount-point'] = '/foreign' with self.assertRaises(SystemExit):host.observe_attachment(self.record) + def held(self): + data = subprocess.check_output(['git', '--no-replace-objects', '-C', str(HERE.parents[2]), 'show', + 'b5cf1e9df9b3ffe88e3fa3290074ae10a8422171:scripts/acceptance/condition-history/guest-host.py']) + module = types.ModuleType('held_b5_guest_source') + exec(compile(data, 'git:held-b5-guardian', 'exec'), module.__dict__) + return module + + def store_fixture(self): + self.scoped['home'].mkdir(parents=True) + self.scoped['image'].write_bytes(b'fixed-image') + self.record['profile'] = self.proposal()['profile'] + self.record['profile_sha256'] = self.put('unused-profile.json', self.record['profile']) + (self.scoped['mount'] / 'guest.yaml').write_bytes(host.canonical(self.record['profile'])) + info = self.scoped['image'].stat() + self.record.update(image_dev=info.st_dev, image_ino=info.st_ino, whole_device='/dev/disk9', + volume_uuid=BOOT, host_boot_id=BOOT) + def native(record, argv): + if 'hdiutil' in argv[0]: + return plistlib.dumps({'images': [{'image-path': str(self.scoped['image']), + 'system-entities': [{'dev-entry': '/dev/disk9', 'mount-point': str(self.scoped['mount'])}]}]}) + if argv[-1] == '/dev/disk9':return plistlib.dumps({'TotalSize': host.STORE_BYTES}) + return plistlib.dumps({'VolumeUUID': BOOT, 'TotalSize': host.STORE_BYTES - 4096, + 'MountPoint': str(self.scoped['mount']), 'FilesystemType': 'apfs'}) + return native + + def test_actual_held_store_check_admits_destination_corruption(self): + old = self.held();native = self.store_fixture() + target = self.scoped['mount'] / 'a/guest.raw.gz';target.parent.mkdir();target.write_bytes(b'old') + self.record['guest_image'] = host.bound_input(target, 3, host.digest(b'old')) + target.write_bytes(b'bad') + with mock.patch.object(old, 'call', side_effect=native):old.verify_store(self.record) + with mock.patch.object(host, 'IMAGE_BYTES', 3), mock.patch.object(host, 'IMAGE_SHA', host.digest(b'old')), \ + self.assertRaisesRegex(SystemExit, 'digest'): + host.verify_guest_inputs(self.record) + replacement = target.with_name('replacement');replacement.write_bytes(b'old');replacement.replace(target) + with mock.patch.object(old, 'call', side_effect=native):old.verify_store(self.record) + with mock.patch.object(host, 'IMAGE_BYTES', 3), mock.patch.object(host, 'IMAGE_SHA', host.digest(b'old')), \ + self.assertRaisesRegex(SystemExit, 'identity'): + host.verify_guest_inputs(self.record) + + def test_actual_held_plan_admits_profile_edit_restore_but_new_create_consumes_captured_stdin(self): + old = self.held();native = self.store_fixture() + legacy = old.plan(str(self.parent), TOKEN, self.record['limactl'], + str(self.scoped['mount'] / 'a/guest.raw.gz'), self.profile) + path = self.scoped['mount'] / 'guest.yaml';reviewed = path.read_bytes() + foreign = json.loads(reviewed);foreign['mounts'] = [{'location': '/foreign/host'}] + path.write_bytes(host.canonical(foreign)) + self.assertEqual(json.loads(pathlib.Path(legacy['create_vm'][-1]).read_bytes())['mounts'], foreign['mounts']) + path.write_bytes(reviewed) + with mock.patch.object(old, 'call', side_effect=native):old.verify_store(self.record) + self.record['guest_image'] = {'device': 1, 'inode': 2, 'bytes': 3, 'sha256': 'f' * 64} + self.put('launch.json', {'record_sha256': 'd' * 64, 'manifest_sha256': self.record['manifest_sha256'], 'action': 'create-and-start'}) + consumed = [] + def command(record, argv, input_data=None): + if 'create' in argv: + path.write_bytes(host.canonical(foreign)) + consumed.append((argv, input_data)) + path.write_bytes(reviewed) + if 'shell' in argv: + return host.canonical({'pid': 100, 'start_ticks': 1000, 'guest_boot_id': BOOT, + 'sha256': self.record['guest_agent_binary_sha256']}) + return b'' + with mock.patch.object(host, 'verify_store'), mock.patch.object(host, 'verify_guest_inputs'), \ + mock.patch.object(host, 'call', side_effect=command), \ + mock.patch.object(host, 'effective_config', side_effect=[{'status': 'Stopped'}, {'status': 'Stopped'}, {'status': 'Running'}]), \ + mock.patch.object(host, 'capture_agent', return_value={'pid': 2345}): + host.launch_guest(self.record, 'd' * 64) + self.assertEqual(consumed[0][0][-1], '-') + self.assertEqual(consumed[0][1], reviewed) + self.assertEqual(json.loads(consumed[0][1])['mounts'], []) + self.assertTrue((self.parent / 'vm-started.json').exists()) + + def test_actual_held_authority_accepts_correct_hash_at_unselected_agent_path(self): + old = self.held();self.store_fixture() + binary = self.temp / 'tools/bin/limactl';binary.parent.mkdir(parents=True);binary.write_bytes(b'launcher') + unrelated = self.temp / 'unselected-agent.gz';unrelated.write_bytes(b'pinned agent bytes') + value = {key: self.record.get(key) for key in old.AUTHORITY_KEYS} + value.update(schema='durable-workflow.condition-host/v1', limactl=str(binary), guest_agent=str(unrelated)) + sha = self.put('legacy-record.json', value) + with mock.patch.object(old, 'LIMA_SHA', host.digest(binary.read_bytes())), \ + mock.patch.object(old, 'AGENT_SHA', host.digest(unrelated.read_bytes())), \ + mock.patch.object(old, 'boot_identity', return_value=BOOT): + accepted = old.authority(self.parent / 'legacy-record.json', sha, self.record['guardian_sha256']) + with self.assertRaisesRegex(SystemExit, 'not the executable-selected'): + host.selected_agent(accepted) + + def test_effective_config_refuses_resource_share_image_and_forwarding_changes(self): + self.record['profile'] = self.proposal()['profile'] + expected = {k: v for k, v in self.record['profile'].items() if k not in ('base', 'minimumLimaVersion')} + value = {'name': self.scoped['name'], 'dir': str(self.scoped['instance']), 'status': 'Stopped', + 'vmType': 'vz', 'arch': 'x86_64', 'cpus': 4, 'memory': host.MEMORY_BYTES, + 'disk': host.GUEST_BYTES, 'config': expected} + with mock.patch.object(host, 'call', return_value=host.canonical(value)): + self.assertEqual(host.effective_config(self.record, 'Stopped'), value) + for field, change in [('mounts', [{'location': '/foreign'}]), ('portForwards', []), + ('images', []), ('ssh', dict(expected['ssh'], forwardAgent=True))]: + changed = copy.deepcopy(value);changed['config'][field] = change + with self.subTest(field=field), mock.patch.object(host, 'call', return_value=host.canonical(changed)), \ + self.assertRaises(SystemExit):host.effective_config(self.record, 'Stopped') + for key, change in [('cpus', 8), ('memory', host.MEMORY_BYTES + 1), ('disk', host.GUEST_BYTES + 1)]: + changed = copy.deepcopy(value);changed[key] = change + with self.subTest(key=key), mock.patch.object(host, 'call', return_value=host.canonical(changed)), \ + self.assertRaises(SystemExit):host.effective_config(self.record, 'Stopped') + + def test_launch_refuses_unbound_request_before_native_create(self): + self.put('launch.json', {'action': 'create-and-start', 'record_sha256': 'foreign'}) + with mock.patch.object(host, 'call', side_effect=AssertionError('must not create')), self.assertRaises(SystemExit): + host.launch_guest(self.record, 'd' * 64) + + def test_actual_held_agent_capture_accepts_unselected_path_but_new_capture_refuses(self): + old = self.held() + self.scoped['instance'].mkdir(parents=True) + self.record['guest_agent'] = '/reviewed/share/lima/lima-guestagent.Linux-x86_64.gz' + (self.scoped['instance'] / 'ha.pid').write_text('2345\n') + process = {'pid': 2345, 'start_seconds': self.record['started_unix'], 'start_microseconds': 123} + command = [self.record['limactl'], 'hostagent', '--pidfile', str(self.scoped['instance'] / 'ha.pid'), + '--guestagent', '/unselected/agent.gz', self.scoped['name']] + import shlex + raw = shlex.join(command).encode() + with mock.patch.object(old, 'call', return_value=raw), mock.patch.object(old, 'process_identity', return_value=process): + self.assertEqual(old.capture_agent(self.record), process) + with mock.patch.object(host, 'call', return_value=raw), mock.patch.object(host, 'process_identity', return_value=process), \ + self.assertRaisesRegex(SystemExit, 'foreign'): + host.capture_agent(self.record) + command[command.index('--guestagent') + 1] = self.record['guest_agent'] + with mock.patch.object(host, 'call', return_value=shlex.join(command).encode()), \ + mock.patch.object(host, 'process_identity', return_value=process): + self.assertEqual(host.capture_agent(self.record), process) + + def test_running_guest_agent_kernel_hash_must_match_pinned_selected_binary(self): + self.store_fixture() + self.record['guest_image'] = {'device': 1, 'inode': 2, 'bytes': 3, 'sha256': 'f' * 64} + self.put('launch.json', {'record_sha256': 'd' * 64, 'manifest_sha256': self.record['manifest_sha256'], 'action': 'create-and-start'}) + def command(record, argv, input_data=None): + if 'shell' in argv:return host.canonical({'pid': 100, 'start_ticks': 1000, 'guest_boot_id': BOOT, 'sha256': 'wrong'}) + return b'' + with mock.patch.object(host, 'verify_store'), mock.patch.object(host, 'verify_guest_inputs'), \ + mock.patch.object(host, 'call', side_effect=command), \ + mock.patch.object(host, 'effective_config', side_effect=[{'status': 'Stopped'}, {'status': 'Stopped'}, {'status': 'Running'}]), \ + mock.patch.object(host, 'capture_agent', return_value={'pid': 2345}), \ + self.assertRaisesRegex(SystemExit, 'running guest agent'): + host.launch_guest(self.record, 'd' * 64) + self.assertTrue((self.parent / 'vm-created.json').exists()) + self.assertFalse((self.parent / 'vm-started.json').exists()) + if __name__ == '__main__': unittest.main() From fc3d2528072f72381c017425f6384d9f928c5d65 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:41:30 +0000 Subject: [PATCH 11/14] test: propose independent one-shot Colima restoration --- .github/workflows/ci.yml | 2 + scripts/acceptance/colima-window/README.md | 105 ++++ .../acceptance/colima-window/test_window.py | 512 +++++++++++++++++ scripts/acceptance/colima-window/window.py | 524 ++++++++++++++++++ 4 files changed, 1143 insertions(+) create mode 100644 scripts/acceptance/colima-window/README.md create mode 100644 scripts/acceptance/colima-window/test_window.py create mode 100644 scripts/acceptance/colima-window/window.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88294fb..43a92f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,8 @@ jobs: python -m unittest discover -s scripts/acceptance/installed-native -p test_cell_lifetime.py - name: Test the condition-history runner without Docker or Cargo run: python -B -m unittest discover -s scripts/acceptance/condition-history -p 'test_*.py' + - name: Test the one-shot Colima window with fake native controls + run: python -B -m unittest discover -s scripts/acceptance/colima-window -p 'test_*.py' - name: Check committed Cargo locks for known advisories run: python scripts/ci/check-dependency-advisories.py diff --git a/scripts/acceptance/colima-window/README.md b/scripts/acceptance/colima-window/README.md new file mode 100644 index 0000000..4e91e77 --- /dev/null +++ b/scripts/acceptance/colima-window/README.md @@ -0,0 +1,105 @@ +# One-shot default-Colima handoff proposal + +This source accompanies #210. It is not admitted for execution. It preserves the +separately reviewed task controller at `df617fb681958801182e67ca63e886041bec6de0` +and its 13-member manifest +`9ce029dcfb64220ff233e9f42605400ea4a73aceb598272a4034b3e8a65d6e38`. +The outer source is a separate review subject. It does not change the Linux +process epochs, guest/network controls, fixed task store or acceptance rules. + +The selected existing Intel macOS/VZ host cannot currently meet the task's +10-GiB available-memory gate while its default Colima profile runs. General owns +the coordinated idle handoff and final execution decision. Security reviews the +source and exact private invocation. No service stop, launchd bootstrap, VM, +disk allocation, pull or consumer build follows from these source tests. + +Run the controls from the repository root: + +```sh +python3 -B -m unittest discover -s scripts/acceptance/colima-window -p 'test_*.py' +``` + +The tests use synthetic native responses and disposable ordinary files. They +exercise readiness, original-state drift, owner death/PID reuse, partial stop, +restoration faults, survivors and exclusive independent auditing. They neither +contact Docker nor load/unload a service or create a VM. + +## Contingent invocation and custody + +A private proposal supplies the exact source digest, host boot/UID, original +LaunchAgent bytes/effective properties and arguments, six launcher hashes, +original profile/configuration hashes, original disk device/inode identities, +local Docker endpoint and idle engine ID/version/architecture/CPU/image/volume +inventory. The supported original service invocation is its existing foreground +`start -f --cpu 4 --memory 6 --disk 60 --vm-type vz --mount-type virtiofs`. +Its existing `KeepAlive={SuccessfulExit:false}` policy is preserved. Files are +never edited, disks never removed, and images/volumes never pruned. Disk contents +can change during an ordinary boot; disk identity, configurations and the engine +inventory must remain unchanged. Usable Linux memory can vary across boots and +must stay within the captured 6-GiB allocation rather than match one page count. + +The proposal leaves the owner epoch null. The foreground native `owner` +invocation fills its own kernel PID/start-microsecond/boot epoch, validates the +proposal, and installs a one-shot user LaunchAgent. That controller has launchd +as its parent, `KeepAlive=false`, `LaunchOnlyOnce=true`, no inherited credentials, +and no output files beyond bounded exclusive JSON receipts. Its source, frozen +SDK source, intent and guest profile are captured arguments. It does not reopen +mutable source or watch-input files. The owner releases `go.json` only after the +actual launchd PID and kernel epoch match the controller's ready receipt. + +Both initial controller bootstrap and later original-service restoration use a +held, unlinked plist descriptor. Read-only native `plutil -lint /dev/fd/N` works +on the selected host. **Actual launchd descriptor consumption is still unproven.** +The initial bootstrap must establish the independent controller before any +original-service stop. Failure has no pathname fallback and authorizes no stop. +The controller rechecks the bound tools, original service/configuration, idle +engine, VM epochs and its own ready receipt immediately before bootout/stop. + +Let `S` be the final admitted outer Unix epoch. No deadline is extended: + +| Stage | Absolute bound | +| --- | --- | +| Owner/watch startup | `S` through `S+30` | +| Original service stop | before `S+120` | +| Frozen task start | `S+120`, at most five seconds late | +| Frozen task active / cleanup | `S+1860` / `S+1920` | +| Original restoration | before `S+2220` | +| Controller removal / final audit | before `S+2280` | + +The frozen task still owns its 300-second setup, 4-vCPU/6-GiB guest, fixed +32-GiB host store, 20-GiB guest store, fresh capacity refusal and unchanged inner +deadlines. This outer controller starts the frozen preparation as its own child, +then consumes its exact ready authority before issuing the create/start request. +No extra VM or dependency operation is added. + +If the foreground owner is hard-stopped, the independent controller retains +custody. Before VM launch it may terminate only its own child and use the +complete frozen cleanup authority. After launch it lets the frozen child reach +its unchanged cleanup cutoff. Restoration requires the child, task VM, image, +mount and instance to be absent; malformed or missing authority cannot substitute +for absence. Any uncertain task cleanup leaves the original service stopped +and calls the existing host `dm` only to report that real fault. Restoration +failure is terminal and also reports the fault. There is no retry or provisioning +fallback. Host reboot or killing the independent controller is outside this +same-boot guarantee and requires operator recovery; no unconditional boot-time +restoration service is installed. + +A normal restoration bootstraps the captured original plist, starts that same +service and independently reads back its effective arguments/properties, all +original file/disk bindings, idle engine fingerprint and inventories. A receipt +is written before the controller removes its own launchd registration. A +separate `audit` invocation must observe the controller process/job and task +absent and independently repeat the original-engine/file checks. Summary and +audit files are exclusive, and a self-removal failure marker refuses acceptance. +Operational window success never substitutes for the frozen SDK acceptance +summary and independent host/guest audit. + +Private invocation is captured from Git, without executing checkout files: +select the reviewed outer commit's `window.py` bytes, the exact frozen commit's +`guest-host.py` bytes and `guest-profile.json`, verify their reviewed hashes, +then exec the pinned native Python with `-I -S -c OUTER_SOURCE owner --record +PRIVATE_PROPOSAL --record-sha256 PROPOSAL_SHA --sdk-code FROZEN_SOURCE --profile +PRIVATE_PROFILE`. Keep the owner foreground TTY open. General receives the exact +argument vector and private contingent record separately. Final host boot, +launcher/cache/configuration/engine identities, owner epoch, capacity and `S` +remain unbound until a coordinated admission. A stale record refuses execution. diff --git a/scripts/acceptance/colima-window/test_window.py b/scripts/acceptance/colima-window/test_window.py new file mode 100644 index 0000000..3275208 --- /dev/null +++ b/scripts/acceptance/colima-window/test_window.py @@ -0,0 +1,512 @@ +"""Inert lifecycle faults. Native service, Docker and VM calls are all fake.""" +import copy +import importlib.util +import json +import os +import pathlib +import plistlib +import shutil +import sys +import tempfile +import time +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('colima_window_controls', HERE / 'window.py') +window = importlib.util.module_from_spec(spec) +spec.loader.exec_module(window) +CODE = (HERE.parent / 'condition-history/guest-host.py').read_text() +SOURCE = window.sha((HERE / 'window.py').read_bytes()) +TOKEN = '0123456789abcdef' +BOOT = '11111111-1111-1111-1111-111111111111' + + +class Child: + pid = 4321 + alive = True + + def poll(self):return None if self.alive else 0 + def terminate(self):self.alive = False + def wait(self, timeout):self.alive = False + + +class WindowControls(unittest.TestCase): + def setUp(self): + self.temp = pathlib.Path(tempfile.mkdtemp(prefix='sw-', dir='/tmp')).resolve() + self.addCleanup(shutil.rmtree, self.temp) + self.root = self.temp / ('sdk-rust-window-' + TOKEN) + self.root.mkdir(mode=0o700) + self.sdk = window.load_sdk(CODE) + self.owner_pid = 3456 + def epoch(pid): + return dict(pid=pid, pgid=pid, uid=os.getuid(), start_seconds=1000, + start_microseconds=123456, boot_id=BOOT) + self.epoch = epoch + self.processes = {pid:epoch(pid) for pid in (self.owner_pid, os.getpid(), 5678)} + self.sdk.process_identity = lambda pid:copy.deepcopy(self.processes.get(pid)) + self.sdk.boot_identity = lambda:BOOT + self.loaded, self.healthy, self.controller = True, True, True + self.calls, self.bootstraps, self.notices = [], [], [] + self.native_fault, self.bad_properties, self.bad_arguments = None, False, False + self.original_files = {} + tools = {} + for name in ('python', 'launchctl', 'plutil', 'colima', 'docker', 'dm'): + path = self.temp / name + path.write_bytes(('fake executable ' + name).encode()) + tools[name] = self.bind(path) + # Native absolute launchctl/plutil paths have synthetic, checked bytes. + for name, path in [('launchctl', '/bin/launchctl'), ('plutil', '/usr/bin/plutil')]: + tools[name]['path'] = path + self.original_files[path] = ('fake executable ' + name).encode() + verify = window.verify_file + def fake_system_file(sdk, binding): + if binding['path'] in self.original_files: + raw = self.original_files[binding['path']] + window.require(set(binding) == {'path', 'bytes', 'sha256'} and len(raw) == binding['bytes'] + and window.sha(raw) == binding['sha256'], 'native tool changed') + else:verify(sdk, binding) + self.addCleanup(mock.patch.stopall) + mock.patch.object(window, 'verify_file', side_effect=fake_system_file).start() + home = self.temp / 'profile' + configs = [] + for suffix in ('default/colima.yaml', '_lima/colima/lima.yaml'): + path = home / suffix;path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'unchanged original configuration\n');configs.append(self.bind(path)) + disk = home / 'default/diffdisk';disk.write_bytes(b'original disk') + info = disk.stat() + plist = self.temp / 'original.plist' + self.original_plist = plistlib.dumps({'Label':'org.example.colima', 'RunAtLoad':True, + 'KeepAlive':{'SuccessfulExit':False}, 'ProgramArguments':[tools['colima']['path'], 'start', '-f', '--cpu', '4', '--memory', '6', '--disk', '60', '--vm-type', 'vz', '--mount-type', 'virtiofs'], + 'EnvironmentVariables':{'COLIMA_HOME':str(home), 'PATH':'/original/bin:/usr/bin:/bin'}}) + plist.write_bytes(self.original_plist) + started = int(time.time()) + task = {key:'unused' for key in self.sdk.INTENT_KEYS} + task.update(schema='durable-workflow.condition-host-setup/v1', token=TOKEN, + runner_commit=window.SDK_R, manifest_sha256=window.SDK_M, guardian_sha256=window.SDK_CODE_SHA, + started_unix=started + 120, active_unix=started + 1860, cleanup_unix=started + 1920, + parent=str(self.temp / ('sr-' + TOKEN)), host_boot_id=BOOT) + self.engine = {'info':{'ID':'original-engine', 'ServerVersion':'29.5.2', 'Architecture':'x86_64', + 'OSType':'linux', 'NCPU':4, 'DockerRootDir':'/var/lib/docker'}, + 'memory_ceiling_bytes':6 * 1024**3, 'images':['sha256:public'], 'volumes':['original-volume']} + self.record = dict(schema='durable-workflow.colima-window/v1', token=TOKEN, source_sha256=SOURCE, + host_boot_id=BOOT, uid=os.getuid(), scope=str(self.root), owner_process=epoch(self.owner_pid), + started_unix=started, sdk_intent=task, tools=tools, profile_home=str(home), + service=dict(label='org.example.colima', plist=self.bind(plist), + arguments=plistlib.loads(self.original_plist)['ProgramArguments'], keep_alive={'SuccessfulExit':False}, + properties='runatload | inferred program', vm_processes=[epoch(5678)]), + config_files=configs, disks=[dict(path=str(disk), device=info.st_dev, inode=info.st_ino)], + docker_host='unix://' + str(home / 'default/docker.sock'), engine=copy.deepcopy(self.engine)) + self.intent_sha = window.sha(window.canonical(self.record)) + self.w = window.Window(self.sdk, self.record, self.intent_sha, CODE) + self.w.call = self.call + self.put('go.json', {'intent_sha256':self.intent_sha, 'action':'one-window'}) + + def bind(self, path): + raw = path.read_bytes() + return dict(path=str(path), bytes=len(raw), sha256=window.sha(raw)) + + def put(self, name, value): + (self.root / name).write_bytes(window.canonical(value)) + + def ready(self): + self.put('window-ready.json', dict(intent_sha256=self.intent_sha, + source_sha256=SOURCE, process=self.epoch(os.getpid()))) + + def call(self, argv, cutoff, *, pass_fds=(), stop=False, check=True): + window.require(cutoff > time.monotonic(), 'fake fixed cutoff expired') + self.calls.append(argv) + name = pathlib.Path(argv[0]).name + if name == self.native_fault:raise SystemExit('injected native fault') + args = argv[1:] + if name == 'launchctl': + if args[0] == 'print': + controller = args[1].endswith(window.job_label(self.record)) + if not (self.controller if controller else self.loaded): + return 113, b'', b'Could not find service' + pid = os.getpid() if controller else 6789 + program = self.record['tools']['python' if controller else 'colima']['path'] + properties = 'wrong' if self.bad_properties else self.record['service']['properties'] + # Nested fields and source text must never become root authority. + return 0, (f'job = {{\n\tprogram = {program}\n\tpid = {pid}\n' + f'\tproperties = {properties}\n\targuments = {{\n\t\tpid = 9999\n' + '\t\tprogram = attacker\n pid = 8888\n\t}\n}\n').encode(), b'' + if args[0] == 'bootout':self.loaded = False + elif args[0] == 'bootstrap': + descriptor = int(args[-1].split('/')[-1]);self.assertIn(descriptor, pass_fds) + os.lseek(descriptor, 0, os.SEEK_SET) + self.bootstraps.append(os.read(descriptor, 512 * 1024)) + self.loaded = True + elif args[0] == 'kickstart':self.healthy = True + elif args[0] == 'remove':self.controller = False + else:self.failTest('unexpected service mutation ' + str(args)) + elif name == 'colima': + self.assertEqual(args, ['--profile', 'default', 'stop']);self.assertTrue(stop) + self.healthy = False;self.processes.pop(5678, None) + elif name == 'ps': + values = ['different'] if self.bad_arguments else self.record['service']['arguments'] + return 0, (' '.join(values)).encode(), b'' + elif name == 'docker': + self.assertEqual(args[:2], ['--host', self.record['docker_host']]) + window.require(self.healthy, 'fake original engine unavailable') + operation = args[2:] + if operation[0] == 'info':return 0, window.canonical(dict(self.engine['info'], MemTotal=6 * 1024**3)), b'' + if operation[0] == 'ps':return 0, b'', b'' + if operation[0] == 'image':return 0, '\n'.join(self.engine['images']).encode(), b'' + if operation[0] == 'volume':return 0, '\n'.join(self.engine['volumes']).encode(), b'' + self.failTest('unexpected Docker operation') + elif name == 'dm':self.notices.append(args[0]) + elif name != 'plutil':self.failTest('unexpected native command ' + name) + return 0, b'', b'' + + def run_fake_task(self, *, survive=False, fail=False): + def start(profile): + self.child = Child();self.w.child = self.child + self.processes.pop(self.owner_pid, None) # hard owner stop after handoff + def monitor(): + if not survive:self.child.alive = False + if fail:raise SystemExit('injected task failure') + self.w.start_task = start;self.w.monitor_task = monitor + self.w.run(SOURCE, {}) + return json.loads((self.root / 'window-result.json').read_bytes()) + + def test_frozen_source_is_hash_bound(self): + self.assertEqual(window.sha(CODE.encode()), window.SDK_CODE_SHA) + with self.assertRaises(SystemExit):window.load_sdk(CODE + '\n') + + def test_complete_private_record_validates(self): + window.validate(self.sdk, self.record, SOURCE) + self.assertEqual(self.w.engine(), self.record['engine']) + + def test_closed_source_epoch_cutoffs_and_scope(self): + mutations = [lambda r:r.update(extra=True), lambda r:r.update(source_sha256='f'*64), + lambda r:r.update(host_boot_id='old-boot'), lambda r:r['owner_process'].update(start_microseconds=1_000_000), + lambda r:r['sdk_intent'].update(runner_commit='f'*40), + lambda r:r['sdk_intent'].update(cleanup_unix=r['sdk_intent']['cleanup_unix']+1), + lambda r:r['tools']['launchctl'].update(path='/tmp/launchctl'), + lambda r:r['service'].update(keep_alive=True), + lambda r:r.update(docker_host='tcp://127.0.0.1:2375'), + lambda r:r.update(scope=str(self.temp)), lambda r:r.update(config_files=[])] + for mutate in mutations: + with self.subTest(mutate=mutate): + candidate = copy.deepcopy(self.record);mutate(candidate) + with self.assertRaises(SystemExit):window.validate(self.sdk, candidate, SOURCE) + + def test_original_tool_config_plist_and_disk_drift_refuse_before_stop(self): + paths = [self.record['tools']['docker']['path'], self.record['config_files'][0]['path'], + self.record['service']['plist']['path'], self.record['disks'][0]['path']] + for name in paths: + with self.subTest(path=name): + path = pathlib.Path(name);raw = path.read_bytes() + if name == self.record['disks'][0]['path']: + moved = path.with_suffix('.held');path.rename(moved);path.write_bytes(raw) + else:path.write_bytes(raw + b'changed') + self.ready() + with self.assertRaises(SystemExit):self.w.stop_default() + self.assertFalse(self.w.stop_attempted) + path.unlink() + if name == self.record['disks'][0]['path']:moved.rename(path) + else:path.write_bytes(raw) + self.assertFalse(any('bootout' in call for call in self.calls)) + + def test_missing_or_tampered_outside_ready_refuses_stop(self): + with self.assertRaises(FileNotFoundError):self.w.stop_default() + self.ready();value = json.loads((self.root/'window-ready.json').read_bytes()) + for key, replacement in [('source_sha256','f'*64), ('intent_sha256','f'*64), ('process',self.epoch(9999))]: + with self.subTest(key=key): + self.put('window-ready.json', dict(value, **{key:replacement})) + with self.assertRaises(SystemExit):self.w.stop_default() + self.assertFalse(self.w.stop_attempted) + + def test_kernel_owner_loss_and_pid_reuse_refuse_stop(self): + self.ready() + for process in (None, dict(self.epoch(self.owner_pid), start_microseconds=123457)): + self.processes[self.owner_pid] = process + self.assertFalse(self.w.owner_alive()) + with self.assertRaises(SystemExit):self.w.stop_default() + self.assertFalse(self.w.stop_attempted) + + def test_effective_service_arguments_properties_and_engine_drift_refuse(self): + self.ready() + for field in ('bad_arguments','bad_properties'): + setattr(self, field, True) + with self.assertRaises(SystemExit):self.w.stop_default() + setattr(self, field, False) + self.engine['images'].append('foreign-image') + with self.assertRaises(SystemExit):self.w.stop_default() + self.assertFalse(self.w.stop_attempted) + + def test_top_level_job_schema_ignores_nested_source_fields(self): + self.assertEqual(self.w.job('org.example.colima')['pid'], 6789) + self.assertEqual(self.w.service_matches()['program'], self.record['tools']['colima']['path']) + + def test_original_vm_epoch_and_fixed_stop_cutoff_refuse(self): + self.ready();self.processes[5678]['start_microseconds'] += 1 + with self.assertRaises(SystemExit):self.w.stop_default() + self.processes[5678] = self.epoch(5678);self.w.stop_cutoff = time.monotonic()-1 + with self.assertRaises(SystemExit):self.w.stop_default() + self.assertFalse(self.w.stop_attempted) + + def test_hard_owner_stop_leaves_independent_restore_and_controller_cleanup(self): + with mock.patch.object(window.subprocess, 'Popen', side_effect=AssertionError('no native experiment')): + result = self.run_fake_task() + self.assertEqual(result['outcome'], 'pending_independent_audit') + self.assertTrue(result['task_absent']);self.assertTrue(self.loaded and self.healthy) + self.assertFalse(self.controller) + self.assertEqual(self.bootstraps, [self.original_plist]) + self.assertFalse(self.notices) + + def test_partial_stop_fault_restores_original_service(self): + self.native_fault = 'colima' + result = self.run_fake_task() + self.assertEqual(result['outcome'], 'fail') + self.assertTrue(result['original_engine_unchanged']);self.assertTrue(self.loaded) + self.assertEqual(self.bootstraps, [self.original_plist]);self.assertFalse(self.controller) + + def test_preflight_failure_still_removes_controller(self): + pathlib.Path(self.record['config_files'][0]['path']).write_bytes(b'drift') + result = self.run_fake_task() + self.assertEqual(result['outcome'], 'fail');self.assertFalse(self.controller) + self.assertFalse(any('bootout' in call for call in self.calls)) + + def test_surviving_child_blocks_restore_and_notifies_operator(self): + result = self.run_fake_task(survive=True) + self.assertEqual(result['outcome'], 'fail');self.assertFalse(self.loaded) + self.assertTrue(result['operator_notified']);self.assertTrue(self.notices[0].startswith('sdk-rust:')) + self.assertFalse(self.bootstraps);self.assertFalse(self.controller) + + def test_task_failure_with_proven_absence_restores_without_claiming_pass(self): + result = self.run_fake_task(fail=True) + self.assertEqual(result['outcome'], 'fail');self.assertTrue(result['task_absent']) + self.assertTrue(self.loaded and self.healthy);self.assertFalse(self.notices) + + def test_restore_consumes_held_plist_not_mutable_path(self): + self.loaded = False + raw = self.original_plist + def mutate_during_lint(argv, cutoff, **kwargs): + if pathlib.Path(argv[0]).name == 'plutil': + pathlib.Path(self.record['service']['plist']['path']).write_bytes(b'edited then restored') + return self.call(argv, cutoff, **kwargs) + self.w.call = mutate_during_lint + self.w.bootstrap_fd(raw) + self.assertEqual(self.bootstraps, [raw]) + + def test_restoration_inventory_mismatch_is_terminal(self): + self.ready();self.w.stop_default() + self.engine['volumes'].append('foreign') + self.w.restore_cutoff = time.monotonic() + 0.02 + with mock.patch.object(window.time, 'sleep'), self.assertRaises(SystemExit):self.w.restore(self.original_plist) + + def test_uncertain_service_query_cannot_authorize_bootstrap(self): + self.w.native = lambda *args, **kwargs:(5,b'',b'permission denied') + with self.assertRaises(SystemExit):self.w.restore(self.original_plist) + self.assertFalse(self.bootstraps) + + def test_task_image_instance_and_recorded_process_survivors_refuse(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'], TOKEN) + scoped['parent'].mkdir(mode=0o700) + for path in (scoped['image'], scoped['instance']): + path.parent.mkdir(parents=True, exist_ok=True);path.write_bytes(b'survivor') + self.assertFalse(self.w.task_absent());path.unlink() + for name,key in [('vm-process.json',None),('host-ready.json','process'),('vm-started.json','host_process')]: + process = self.epoch(5678) + path = scoped['parent']/name;path.write_bytes(window.canonical({key:process} if key else process)) + self.assertFalse(self.w.task_absent());path.unlink() + (scoped['parent']/'host-ready.json').write_bytes(b'not JSON') + with self.assertRaises(ValueError):self.w.task_absent() + + def test_prelaunch_owner_loss_aborts_owned_child_then_frozen_cleanup(self): + self.w.child = Child();self.w.task_record = {'frozen':'authority'} + self.processes.pop(self.owner_pid) + with mock.patch.object(self.sdk, 'cleanup') as cleanup: + self.w.monitor_task() + cleanup.assert_called_once_with(self.w.task_record) + self.assertIsNotNone(self.w.child.poll());self.assertFalse(self.w.launched) + + def test_postlaunch_owner_loss_preserves_absolute_child_cleanup(self): + self.w.child = Child();self.w.task_record = {};self.w.launched = True + self.processes.pop(self.owner_pid) + def finish(_):self.w.child.alive = False + with mock.patch.object(self.sdk, 'cleanup') as cleanup, mock.patch.object(window.time,'sleep',side_effect=finish): + self.w.monitor_task() + cleanup.assert_not_called();self.assertTrue(self.w.task_absent()) + + def test_live_owner_launches_exact_frozen_request_once(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'],TOKEN);scoped['parent'].mkdir() + self.w.child = Child();self.w.task_record = {};self.w.task_record_sha = 'd'*64 + def finish(_):self.w.child.alive = False + with mock.patch.object(window.time,'sleep',side_effect=finish):self.w.monitor_task() + self.assertEqual(json.loads((scoped['parent']/'launch.json').read_bytes()), + dict(record_sha256='d'*64,manifest_sha256=window.SDK_M,action='create-and-start')) + + def test_foreign_child_readiness_refuses_before_authority(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'],TOKEN);scoped['parent'].mkdir() + self.w.child = Child() + (scoped['parent']/'host-ready.json').write_bytes(window.canonical(dict(process=self.epoch(9999), + source_sha256=window.SDK_CODE_SHA, record_sha256='d'*64))) + with mock.patch.object(self.sdk,'authority') as authority, self.assertRaises(SystemExit):self.w.capture_ready() + authority.assert_not_called() + + def test_result_overwrite_refuses_and_still_removes_controller(self): + self.put('window-result.json', {'forged':'pass'}) + with self.assertRaises(FileExistsError):self.run_fake_task() + self.assertEqual(json.loads((self.root/'window-result.json').read_bytes()),{'forged':'pass'}) + self.assertFalse(self.controller) + + def completed(self): + self.run_fake_task();self.processes.pop(os.getpid()) + + def test_independent_final_audit_passes_and_is_exclusive(self): + self.completed();window.audit(self.w,SOURCE) + self.assertEqual(json.loads((self.root/'window-audit.json').read_bytes())['outcome'],'pass') + with self.assertRaises(FileExistsError):window.audit(self.w,SOURCE) + + def test_final_audit_refuses_controller_job_task_and_engine_survivors(self): + self.completed() + for fault in ('process','job','task','engine','marker','digest'): + with self.subTest(fault=fault): + if fault=='process':self.processes[os.getpid()] = self.epoch(os.getpid()) + elif fault=='job':self.controller = True + elif fault=='task':self.w.child.alive = True + elif fault=='engine':self.engine['images'].append('drift') + elif fault=='marker':self.put('controller-cleanup-failure.json',{'outcome':'fail'}) + else: + saved = (self.root/'window-result.json').read_bytes() + value=json.loads(saved);value['intent_sha256']='foreign';self.put('window-result.json',value) + with self.assertRaises(SystemExit):window.audit(self.w,SOURCE) + self.assertFalse((self.root/'window-audit.json').exists()) + self.processes.pop(os.getpid(),None);self.controller=False;self.w.child.alive=False + self.engine['images'][:]=['sha256:public'] + (self.root/'controller-cleanup-failure.json').unlink(missing_ok=True) + if fault=='digest':(self.root/'window-result.json').write_bytes(saved) + + def test_controller_remove_failure_records_fault_and_prevents_acceptance(self): + call=self.call + def broken(argv, cutoff, **kwargs): + if 'remove' in argv:raise SystemExit('self removal failed') + return call(argv,cutoff,**kwargs) + self.w.call=broken + with self.assertRaises(SystemExit):self.run_fake_task() + self.assertTrue((self.root/'controller-cleanup-failure.json').exists()) + self.assertTrue(self.notices) + with self.assertRaises(SystemExit):window.audit(self.w,SOURCE) + + def test_owner_binds_one_shot_launchd_job_and_releases_only_exact_ready(self): + proposal=copy.deepcopy(self.record);proposal['owner_process']=None + proposal['tools']['python']['path']=str(pathlib.Path(sys.executable).resolve()) + (self.root/'go.json').unlink() + captured=[] + def bootstrap(raw): + job=plistlib.loads(raw);captured.append(job) + argv=job['ProgramArguments'] + record=json.loads(argv[argv.index('--intent-json')+1]) + digest=argv[argv.index('--record-sha256')+1] + self.put('window-ready.json',dict(intent_sha256=digest,source_sha256=SOURCE,process=self.epoch(4321))) + self.assertEqual(record['owner_process'],self.epoch(os.getpid())) + self.assertFalse((self.root/'go.json').exists()) + self.processes[4321]=self.epoch(4321) + self.w.bootstrap_fd=bootstrap + self.w.job=mock.Mock(return_value={'pid':4321}) + args=type('Args',(),{'sdk_code':CODE,'profile':str(self.root/'profile')})() + profile=(HERE.parent/'condition-history/guest-profile.json').read_bytes() + with mock.patch.object(window.os,'isatty',return_value=True), mock.patch.object(window.sys,'platform','darwin'), \ + mock.patch.object(window,'validate'), mock.patch.object(window,'Window',return_value=self.w), \ + mock.patch.object(window.selectors,'DefaultSelector',side_effect=SystemExit('test after go')), \ + mock.patch('builtins.print'), self.assertRaisesRegex(SystemExit,'test after go'): + window.owner(self.sdk,proposal,'a'*64,args,(HERE/'window.py').read_text(),profile) + job=captured[0] + self.assertFalse(job['KeepAlive']);self.assertTrue(job['RunAtLoad'] and job['LaunchOnlyOnce']) + self.assertFalse(job['AbandonProcessGroup']) + self.assertEqual(job['StandardOutPath'],'/dev/null');self.assertEqual(job['StandardErrorPath'],'/dev/null') + self.assertEqual(job['ProgramArguments'][1:4],['-I','-S','-c']) + self.assertIn('--intent-json',job['ProgramArguments']);self.assertIn('--profile-json',job['ProgramArguments']) + self.assertTrue((self.root/'go.json').exists()) + self.assertEqual(json.loads((self.root/'docker-config/config.json').read_bytes()),{'auths':{}}) + + def test_late_audit_does_not_extend_absolute_cutoffs(self): + later = self.record['sdk_intent']['cleanup_unix'] + 500 + with mock.patch.object(window.time, 'time', return_value=later): + subject = window.Window(self.sdk, self.record, self.intent_sha, CODE) + self.assertLessEqual(subject.cleanup_cutoff, time.monotonic()) + self.assertLessEqual(subject.restore_cutoff, time.monotonic()) + + def test_watch_validation_failure_removes_its_own_registered_job(self): + args = ['watch', '--record', str(self.root/'unused'), '--record-sha256', self.intent_sha, + '--sdk-code', CODE, '--profile', str(self.root/'unused-profile'), + '--intent-json', window.canonical(self.record).decode(), + '--profile-json', (HERE.parent/'condition-history/guest-profile.json').read_text()] + self.w.job = mock.Mock(return_value={'pid':os.getpid()}) + with mock.patch.object(window.sys, 'orig_argv', ['python','-I','-S','-c',(HERE/'window.py').read_text()]), \ + mock.patch.object(window.sys, 'platform', 'darwin'), mock.patch.object(window.os, 'getppid', return_value=1), \ + mock.patch.object(window, 'load_sdk', return_value=self.sdk), \ + mock.patch.object(window, 'Window', return_value=self.w), \ + mock.patch.object(window, 'validate', side_effect=SystemExit('original drift')), self.assertRaises(SystemExit): + window.main(args) + self.assertFalse(self.controller) + self.assertFalse(any('bootout' in call for call in self.calls)) + + def test_watch_refuses_agent_parent_before_native_service_actions(self): + args = ['watch', '--record', str(self.root/'unused'), '--record-sha256', self.intent_sha, + '--sdk-code', CODE, '--profile', str(self.root/'unused-profile'), + '--intent-json', window.canonical(self.record).decode(), + '--profile-json', (HERE.parent/'condition-history/guest-profile.json').read_text()] + with mock.patch.object(window.sys, 'orig_argv', ['python','-I','-S','-c',(HERE/'window.py').read_text()]), \ + mock.patch.object(window.sys, 'platform', 'darwin'), mock.patch.object(window.os,'getppid',return_value=9999), \ + mock.patch.object(window, 'load_sdk',return_value=self.sdk), \ + mock.patch.object(window, 'Window',return_value=self.w), self.assertRaises(SystemExit):window.main(args) + self.assertFalse(self.calls) + + def test_captured_watch_does_not_reopen_mutable_record_or_profile(self): + profile = (HERE.parent/'condition-history/guest-profile.json').read_text() + args = ['watch', '--record', str(self.root/'missing'), '--record-sha256', self.intent_sha, + '--sdk-code', CODE, '--profile', str(self.root/'missing-profile'), + '--intent-json', window.canonical(self.record).decode(), '--profile-json', profile] + self.w.job = mock.Mock(return_value={'pid':os.getpid()}) + self.record['tools']['python']['path'] = str(pathlib.Path(sys.executable).resolve()) + captured = window.canonical(self.record).decode() + args[args.index('--intent-json')+1] = captured + args[args.index('--record-sha256')+1] = window.sha(captured.encode()) + with mock.patch.object(window.sys,'orig_argv',['python','-I','-S','-c',(HERE/'window.py').read_text()]), \ + mock.patch.object(window.sys,'platform','darwin'), mock.patch.object(window.os,'getppid',return_value=1), \ + mock.patch.object(window,'load_sdk',return_value=self.sdk), mock.patch.object(window,'Window',return_value=self.w), \ + mock.patch.object(window,'validate') as validate, mock.patch.object(self.w,'run') as run: + window.main(args) + validate.assert_called_once();run.assert_called_once_with(SOURCE,json.loads(profile)) + + def test_late_task_start_refuses_before_popen(self): + self.record['sdk_intent']['started_unix'] = int(time.time()) - 10 + with mock.patch.object(window.subprocess,'Popen') as popen, self.assertRaises(SystemExit):self.w.start_task({}) + popen.assert_not_called() + + def test_restoration_fault_notifies_without_retry_or_pass(self): + call=self.call + def broken(argv,cutoff,**kwargs): + if 'bootstrap' in argv:raise SystemExit('restoration bootstrap failed') + return call(argv,cutoff,**kwargs) + self.w.call=broken + result=self.run_fake_task() + self.assertEqual(result['outcome'],'fail');self.assertTrue(result['operator_notified']) + self.assertFalse(self.loaded);self.assertFalse(self.controller) + self.assertEqual(len([x for x in self.calls if 'bootout' in x]),1) + self.assertFalse(self.bootstraps) + + def test_original_file_symlink_hardlink_and_writable_input_refuse(self): + path=self.temp/'input';path.write_bytes(b'original') + binding=self.bind(path) + link=self.temp/'hardlink';os.link(path,link) + with self.assertRaises(SystemExit):window.verify_file(self.sdk,binding) + link.unlink();path.chmod(0o666) + with self.assertRaises(SystemExit):window.verify_file(self.sdk,binding) + path.chmod(0o600);link.symlink_to(path) + with self.assertRaises(SystemExit):window.verify_file(self.sdk,dict(binding,path=str(link))) + + def test_minimal_environment_excludes_forwarded_credentials_and_default_lima_home(self): + with mock.patch.dict(os.environ,{'SSH_AUTH_SOCK':'secret','DOCKER_AUTH_CONFIG':'secret','LIMA_HOME':'foreign'}): + env=self.w.env();stop_env=self.w.env(stop=True) + self.assertEqual(set(env),{'PATH','HOME','TMPDIR','DOCKER_CONFIG'}) + self.assertEqual(stop_env['COLIMA_HOME'],self.record['profile_home']) + self.assertEqual(stop_env['PATH'],'/original/bin:/usr/bin:/bin') + self.assertNotIn('LIMA_HOME',stop_env) + + +if __name__=='__main__':unittest.main() diff --git a/scripts/acceptance/colima-window/window.py b/scripts/acceptance/colima-window/window.py new file mode 100644 index 0000000..07d95f6 --- /dev/null +++ b/scripts/acceptance/colima-window/window.py @@ -0,0 +1,524 @@ +"""One-shot shared-Colima window proposal. No native operation is admitted. + +The separately frozen task controller remains df617fb. This outer subject owns +the original service handoff, restoration, and its own launchd registration. +""" +import argparse +import hashlib +import json +import os +import pathlib +import plistlib +import re +import selectors +import shlex +import signal +import stat +import subprocess +import sys +import tempfile +import time +import types + +SDK_R = 'df617fb681958801182e67ca63e886041bec6de0' +SDK_M = '9ce029dcfb64220ff233e9f42605400ea4a73aceb598272a4034b3e8a65d6e38' +SDK_CODE_SHA = 'bf9dbd8166c71dd1b87bc5761640ce26d07e624d56b19d26e40e5ff02eeab9a1' +KEYS = {'schema', 'token', 'source_sha256', 'host_boot_id', 'uid', 'scope', 'owner_process', + 'started_unix', 'sdk_intent', 'tools', 'service', 'config_files', 'disks', + 'docker_host', 'engine', 'profile_home'} + + +def require(ok, message): + if not ok: + raise SystemExit(message) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def canonical(value): + return (json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n').encode() + + +def load_sdk(code): + require(sha(code.encode()) == SDK_CODE_SHA, 'wrong frozen task controller') + module = types.ModuleType('frozen_df617fb_guest') + exec(compile(code, 'git:' + SDK_R + ':guest-host.py', 'exec'), module.__dict__) + return module + + +def root_of(record): + return pathlib.Path(record['scope']) + + +def job_label(record): + return 'sdk-rust-colima-window-' + record['token'] + + +def target(record, label=None): + return 'gui/' + str(record['uid']) + '/' + (label or job_label(record)) + + +def verify_file(sdk, binding): + require(set(binding) == {'path', 'bytes', 'sha256'}, 'wrong file binding') + path = pathlib.Path(binding['path']) + require(path.is_absolute() and path.resolve() == path, 'noncanonical original input') + require(type(binding['bytes']) is int and 0 < binding['bytes'] <= 64 * 1024**2, 'invalid original input bound') + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_uid in (0, os.getuid()) + and before.st_nlink == 1 and not before.st_mode & 0o022, 'unsafe original input') + sdk.input_bytes(path, binding['bytes'], binding['sha256']) + + +def verify_original(sdk, record): + for binding in [*record['tools'].values(), *record['config_files']]: + verify_file(sdk, binding) + for disk in record['disks']: + require(set(disk) == {'path', 'device', 'inode'}, 'wrong original disk binding') + path = pathlib.Path(disk['path']) + info = path.lstat() + require(path.is_absolute() and path.resolve() == path and stat.S_ISREG(info.st_mode) + and info.st_nlink == 1 and info.st_uid == record['uid'] + and (info.st_dev, info.st_ino) == (disk['device'], disk['inode']), 'original disk identity changed') + service = record['service'] + verify_file(sdk, service['plist']) + raw = sdk.regular(pathlib.Path(service['plist']['path']), 64 * 1024) + require(sha(raw) == service['plist']['sha256'], 'original LaunchAgent changed') + value = plistlib.loads(raw) + require(value['Label'] == service['label'] and value['RunAtLoad'] is True + and value['KeepAlive'] == service['keep_alive'] + and value['ProgramArguments'] == service['arguments'] + and pathlib.Path(value['ProgramArguments'][0]).resolve() == pathlib.Path(record['tools']['colima']['path']), + 'original LaunchAgent configuration changed') + actual_home = value.get('EnvironmentVariables', {}).get('COLIMA_HOME', str(pathlib.Path.home() / '.colima')) + require(actual_home == record['profile_home'], 'original default profile changed') + arguments = value['ProgramArguments'] + require(all(isinstance(item, str) and not any(ch.isspace() for ch in item) + for item in value['ProgramArguments']), 'unsupported service argument shape') + expected = [arguments[0], 'start', '-f', '--cpu', '4', '--memory', '6', '--disk', '60', + '--vm-type', 'vz', '--mount-type', 'virtiofs'] + require(arguments == expected, 'unsupported original default-service invocation') + environment = value.get('EnvironmentVariables', {}) + require(set(environment) <= {'PATH', 'HOME', 'COLIMA_HOME'} + and environment.get('HOME', str(pathlib.Path.home())) == str(pathlib.Path.home()), + 'unexpected original service environment') + return raw + + +def validate(sdk, record, source_sha): + require(set(record) == KEYS and record['schema'] == 'durable-workflow.colima-window/v1', 'wrong window schema') + require(record['source_sha256'] == source_sha and re.fullmatch('[0-9a-f]{16}', record['token']), 'wrong window source/token') + require(type(record['uid']) is int and record['uid'] == os.getuid() and record['uid'] > 0 + and record['host_boot_id'] == sdk.boot_identity(), 'wrong window host/UID') + root = root_of(record) + info = root.lstat() + require(root.is_absolute() and root.resolve() == root and root.name == 'sdk-rust-window-' + record['token'] + and stat.S_ISDIR(info.st_mode) and info.st_uid == record['uid'] + and stat.S_IMODE(info.st_mode) == 0o700, 'wrong private window scope') + require(type(record['started_unix']) is int, 'missing absolute window epoch') + owner = record['owner_process'] + require(set(owner) == {'pid', 'pgid', 'uid', 'start_seconds', 'start_microseconds', 'boot_id'} + and all(type(owner[key]) is int for key in ('pid', 'pgid', 'uid', 'start_seconds', 'start_microseconds')) + and owner['pid'] > 1 and owner['pgid'] > 0 and owner['uid'] == record['uid'] + and owner['start_seconds'] > 0 and 0 <= owner['start_microseconds'] < 1_000_000 + and owner['boot_id'] == record['host_boot_id'], 'wrong native owner epoch') + task = record['sdk_intent'] + require(set(task) == sdk.INTENT_KEYS and task['runner_commit'] == SDK_R + and task['manifest_sha256'] == SDK_M and task['guardian_sha256'] == SDK_CODE_SHA + and task['host_boot_id'] == record['host_boot_id'] and task['token'] == record['token'] + and task['started_unix'] == record['started_unix'] + 120 + and task['active_unix'] == task['started_unix'] + 1740 + and task['cleanup_unix'] == task['started_unix'] + 1800, 'wrong frozen task/window cutoffs') + sdk.paths(task['parent'], task['token']) + require(record['docker_host'].startswith('unix:///') and pathlib.Path(record['profile_home']).is_absolute(), 'local original profile required') + home = pathlib.Path(record['profile_home']) + require(home.resolve() == home and record['docker_host'] == 'unix://' + str(home / 'default/docker.sock'), 'wrong default engine endpoint') + require(set(record['tools']) == {'python', 'launchctl', 'plutil', 'colima', 'docker', 'dm'}, 'wrong window tool set') + require(record['tools']['launchctl']['path'] == '/bin/launchctl' + and record['tools']['plutil']['path'] == '/usr/bin/plutil', 'native service tools required') + service = record['service'] + require(set(service) == {'label', 'plist', 'arguments', 'keep_alive', 'properties', 'vm_processes'} + and service['label'] != job_label(record) and not service['label'].startswith('sdk-rust-') + and re.fullmatch('[A-Za-z0-9_.-]+', service['label']) + and service['keep_alive'] == {'SuccessfulExit': False}, 'unsupported original service policy') + require(service['vm_processes'] and all(process['boot_id'] == record['host_boot_id'] + and process['uid'] == record['uid'] for process in service['vm_processes']), 'original VM process authority required') + require(record['config_files'] and record['disks'], 'original profile/disk inventory required') + require(len(record['config_files']) <= 8 and len(record['disks']) <= 8, 'oversize original inventory') + require({str(home / 'default/colima.yaml'), str(home / '_lima/colima/lima.yaml')} + <= {binding['path'] for binding in record['config_files']}, 'original profile configurations are missing') + engine = record['engine'] + require(set(engine) == {'info', 'memory_ceiling_bytes', 'images', 'volumes'} + and set(engine['info']) == {'ID', 'ServerVersion', 'Architecture', 'OSType', 'NCPU', 'DockerRootDir'} + and engine['info']['NCPU'] == 4 and engine['info']['OSType'] == 'linux' + and engine['info']['Architecture'] in ('x86_64', 'amd64') + and engine['memory_ceiling_bytes'] == 6 * 1024**3 + and all(isinstance(item, str) for key in ('images', 'volumes') for item in engine[key]) + and all(engine[key] == sorted(set(engine[key])) for key in ('images', 'volumes')), + 'wrong original engine fingerprint') + verify_original(sdk, record) + + +class Window: + def __init__(self, sdk, record, intent_sha, sdk_code): + self.sdk, self.record, self.intent_sha, self.sdk_code = sdk, record, intent_sha, sdk_code + self.root = root_of(record) + now, tick = time.time(), time.monotonic() + self.stop_cutoff = tick + max(0, record['started_unix'] + 120 - now) + self.task_cutoff = tick + max(0, record['sdk_intent']['cleanup_unix'] - now) + self.restore_cutoff = tick + max(0, record['sdk_intent']['cleanup_unix'] + 300 - now) + self.cleanup_cutoff = tick + max(0, record['sdk_intent']['cleanup_unix'] + 360 - now) + self.child = None + self.task_record = None + self.task_record_sha = None + self.launched = False + self.stop_attempted = False + self.original_plist = None + + def env(self, stop=False): + result = {'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'HOME': str(pathlib.Path.home()), + 'TMPDIR': str(self.root), 'DOCKER_CONFIG': str(self.root / 'docker-config')} + if stop: + raw = self.original_plist or verify_original(self.sdk, self.record) + result['PATH'] = plistlib.loads(raw).get('EnvironmentVariables', {}).get('PATH', result['PATH']) + result['COLIMA_HOME'] = self.record['profile_home'] + return result + + def call(self, argv, cutoff, *, pass_fds=(), stop=False, check=True): + remaining = cutoff - time.monotonic() + require(remaining > 0, 'window command cutoff') + child = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, pass_fds=pass_fds, env=self.env(stop), close_fds=True) + output, errors = bytearray(), bytearray() + end = time.monotonic() + min(120 if stop else 20, remaining) + try: + with selectors.DefaultSelector() as selector: + selector.register(child.stdout, selectors.EVENT_READ, output) + selector.register(child.stderr, selectors.EVENT_READ, errors) + while selector.get_map(): + require(time.monotonic() < end, 'window native command timed out') + for key, _ in selector.select(min(0.1, max(0, end - time.monotonic()))): + data = os.read(key.fileobj.fileno(), 65536) + if not data:selector.unregister(key.fileobj);continue + require(len(output) + len(errors) + len(data) <= 2 * 1024**2, 'oversize native window response') + key.data.extend(data) + child.wait(timeout=max(0.001, end - time.monotonic())) + require(not check or child.returncode == 0, 'window native control failed') + return child.returncode, bytes(output), bytes(errors) + finally: + if child.poll() is None:child.kill();child.wait(timeout=2) + child.stdout.close();child.stderr.close() + + def native(self, name, *argv, cutoff=None, **kwargs): + verify_file(self.sdk, self.record['tools'][name]) + return self.call([self.record['tools'][name]['path'], *argv], cutoff or self.restore_cutoff, **kwargs) + + def job(self, label, cutoff=None): + code, output, error = self.native('launchctl', 'print', target(self.record, label), cutoff=cutoff, check=False) + if code: + require(b'Could not find service' in error, 'uncertain launchd query') + return None + text = output.decode() + def one(name, optional=False): + # Only top-level fields, never a similarly named line inside the + # controller's multiline -c argument or an environment block. + matches = re.findall(r'^\t' + name + r' = (.+)$', text, re.MULTILINE) + if optional and not matches:return None + require(len(matches) == 1, 'unexpected launchd effective schema') + return matches[0].strip() + pid = one('pid', optional=True) + return {'pid': int(pid) if pid else None, 'program': one('program'), 'properties': one('properties')} + + def engine(self): + def docker(*args):return self.native('docker', '--host', self.record['docker_host'], *args)[1].decode().strip() + value = json.loads(docker('info', '--format', '{{json .}}')) + require(not docker('ps', '-aq'), 'original engine is no longer idle') + # Usable Linux MemTotal can vary slightly across a boot; the captured + # original service/config fixes its 6-GiB allocation, not that page count. + require(5 * 1024**3 <= value['MemTotal'] <= 6 * 1024**3, 'original engine memory is outside its profile') + return {'info': {key: value[key] for key in ('ID', 'ServerVersion', 'Architecture', 'OSType', 'NCPU', 'DockerRootDir')}, + 'memory_ceiling_bytes': 6 * 1024**3, + 'images': sorted(set(docker('image', 'ls', '--no-trunc', '--format', '{{.ID}}').splitlines())), + 'volumes': sorted(docker('volume', 'ls', '--format', '{{.Name}}').splitlines())} + + def service_matches(self): + job = self.job(self.record['service']['label']) + require(job and job['pid'] and pathlib.Path(job['program']).resolve() == pathlib.Path(self.record['tools']['colima']['path']) + and job['properties'] == self.record['service']['properties'], 'effective original service changed') + command = self.call(['/bin/ps', '-ww', '-p', str(job['pid']), '-o', 'command='], self.restore_cutoff)[1] + require(shlex.split(command.decode()) == self.record['service']['arguments'], 'effective original service arguments changed') + return job + + def bootstrap_fd(self, raw): + # An unlinked descriptor holds the captured plist; no edited pathname + # can replace the LaunchAgent configuration consumed by launchctl. + with tempfile.TemporaryFile(dir=self.root) as stream: + stream.write(raw);stream.flush();stream.seek(0) + descriptor = '/dev/fd/' + str(stream.fileno()) + self.native('plutil', '-lint', descriptor, pass_fds=(stream.fileno(),)) + self.native('launchctl', 'bootstrap', 'gui/' + str(self.record['uid']), descriptor, pass_fds=(stream.fileno(),)) + + def owner_alive(self): + expected = self.record['owner_process'] + return self.sdk.process_identity(expected['pid']) == expected + + def stop_default(self): + require(self.owner_alive(), 'owner disappeared before stop') + ready = json.loads(self.sdk.regular(self.root / 'window-ready.json')) + require(ready['intent_sha256'] == self.intent_sha + and ready['source_sha256'] == self.record['source_sha256'] + and ready['process'] == self.sdk.process_identity(os.getpid()), 'outside-agent readiness changed') + verify_original(self.sdk, self.record) + self.service_matches() + require(self.engine() == self.record['engine'], 'original engine inventory changed') + require(all(self.sdk.process_identity(process['pid']) == process + for process in self.record['service']['vm_processes']), 'original VM process changed') + require(time.monotonic() < self.stop_cutoff, 'stop window expired before mutation') + self.stop_attempted = True + self.native('launchctl', 'bootout', target(self.record, self.record['service']['label']), cutoff=self.stop_cutoff) + self.native('colima', '--profile', 'default', 'stop', cutoff=self.stop_cutoff, stop=True) + require(self.job(self.record['service']['label'], self.stop_cutoff) is None, 'original service survived stop') + require(all(self.sdk.process_identity(process['pid']) is None + for process in self.record['service']['vm_processes']), 'original VM survived stop') + self.sdk.exclusive(self.root / 'default-stopped.json', {'intent_sha256': self.intent_sha}) + + def start_task(self, profile): + while time.time() < self.record['sdk_intent']['started_unix']: + require(self.owner_alive() and time.monotonic() < self.stop_cutoff, 'owner/start window lost') + time.sleep(0.1) + require(self.owner_alive() and time.time() < self.record['sdk_intent']['started_unix'] + 5, + 'owner/start window lost before task') + sdk_intent = self.record['sdk_intent'] + intent_sha = self.sdk.exclusive(self.root / 'sdk-intent.json', sdk_intent) + self.sdk.exclusive(self.root / 'guest-profile.json', profile) + argv = [self.record['tools']['python']['path'], '-I', '-S', '-c', self.sdk_code, 'prepare', + '--record', str(self.root / 'sdk-intent.json'), '--record-sha256', intent_sha, + '--profile', str(self.root / 'guest-profile.json')] + self.child = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True, env=self.env()) + + def capture_ready(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'], self.record['token']) + ready_path = scoped['parent'] / 'host-ready.json' + if not ready_path.exists():return False + ready = json.loads(self.sdk.regular(ready_path)) + require(self.child and ready['process']['pid'] == self.child.pid + and self.sdk.process_identity(self.child.pid) == ready['process'] + and ready['source_sha256'] == SDK_CODE_SHA, 'task readiness is not the owned child') + self.task_record_sha = ready['record_sha256'] + self.task_record = self.sdk.authority(scoped['parent'] / 'host-record.json', self.task_record_sha, SDK_CODE_SHA) + require(self.task_record['runner_commit'] == SDK_R and self.task_record['manifest_sha256'] == SDK_M, + 'task readiness tuple changed') + require(self.task_record['parent'] == self.record['sdk_intent']['parent'] + and self.task_record['token'] == self.record['token'], 'task readiness scope changed') + return True + + def task_absent(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'], self.record['token']) + if self.child and self.child.poll() is None:return False + if os.path.lexists(scoped['image']) or os.path.ismount(scoped['mount']) or os.path.lexists(scoped['instance']):return False + for filename, key in (('vm-process.json', None), ('vm-started.json', 'host_process'), ('host-ready.json', 'process')): + path = scoped['parent'] / filename + if path.exists(): + process = json.loads(self.sdk.regular(path)) + if key:process = process[key] + if self.sdk.process_identity(process['pid']) is not None:return False + return True + + def monitor_task(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'], self.record['token']) + while self.child.poll() is None and time.monotonic() < self.task_cutoff: + ready = self.task_record is not None or self.capture_ready() + if ready and not self.launched and self.owner_alive(): + self.sdk.exclusive(scoped['parent'] / 'launch.json', {'record_sha256': self.task_record_sha, + 'manifest_sha256': SDK_M, 'action': 'create-and-start'}) + self.launched = True + # No owner is needed for the frozen child's absolute-cutoff cleanup. + # Before VM launch, a complete ready record also permits early abort. + if ready and not self.launched and not self.owner_alive() and time.monotonic() + 30 < self.task_cutoff: + self.child.terminate();self.child.wait(timeout=5) + self.sdk.cleanup(self.task_record) + break + time.sleep(0.25) + require(self.task_absent(), 'task cleanup is uncertain; original service must remain stopped') + + def restore(self, original_plist): + require(self.task_absent(), 'task survivor blocks restoration') + verify_original(self.sdk, self.record) + if self.job(self.record['service']['label']) is None:self.bootstrap_fd(original_plist) + self.native('launchctl', 'kickstart', target(self.record, self.record['service']['label'])) + while time.monotonic() < self.restore_cutoff: + try: + self.service_matches() + require(self.engine() == self.record['engine'], 'restored engine differs from original') + verify_original(self.sdk, self.record) + return {'original_engine_unchanged': True, 'task_absent': True} + except (SystemExit, OSError, ValueError, KeyError): + time.sleep(0.5) + raise SystemExit('original service restoration is uncertain') + + def notify_failure(self): + # The host's existing local dm cache is used only for a real fault. + # No credential or vault value is copied into the task guest or a log. + try: + self.native('dm', 'sdk-rust: default Colima window has unresolved task/restoration custody; inspect the private window receipt.', cutoff=self.cleanup_cutoff) + return True + except BaseException:return False + + def run(self, source_sha, profile): + original = None + result = {'outcome': 'fail', 'intent_sha256': self.intent_sha, 'source_sha256': source_sha} + try: + original = verify_original(self.sdk, self.record) + self.original_plist = original + self.sdk.exclusive(self.root / 'window-ready.json', {'intent_sha256': self.intent_sha, + 'source_sha256': source_sha, 'process': self.sdk.process_identity(os.getpid())}) + while not (self.root / 'go.json').exists(): + require(self.owner_alive() and time.monotonic() < self.stop_cutoff, 'owner/go window lost') + time.sleep(0.1) + require(json.loads(self.sdk.regular(self.root / 'go.json')) == {'intent_sha256': self.intent_sha, 'action': 'one-window'}, 'wrong window request') + self.stop_default() + self.start_task(profile) + self.monitor_task() + result.update(self.restore(original), outcome='pending_independent_audit') + except BaseException as error: + result['error_type'] = type(error).__name__ + if self.stop_attempted: + try:result.update(self.restore(original)) + except BaseException:result['operator_notified'] = self.notify_failure() + finally: + try: + result['controller_process'] = self.sdk.process_identity(os.getpid()) + self.sdk.exclusive(self.root / 'window-result.json', result) + finally: + self.remove_controller() + + def remove_controller(self): + # Receipt precedes self-removal. A separate invocation must prove that + # this job/process is gone; this process cannot certify its own absence. + try: + self.native('launchctl', 'remove', job_label(self.record), cutoff=self.cleanup_cutoff) + except BaseException: + self.sdk.exclusive(self.root / 'controller-cleanup-failure.json', { + 'outcome': 'fail', 'intent_sha256': self.intent_sha, 'operator_notified': self.notify_failure()}) + raise + + +def audit(window, source_sha): + sdk, record = window.sdk, window.record + result = json.loads(sdk.regular(root_of(record) / 'window-result.json')) + require(result['intent_sha256'] == window.intent_sha and result['source_sha256'] == source_sha + and result['outcome'] == 'pending_independent_audit' and result['task_absent'] is True + and result['original_engine_unchanged'] is True, 'window did not complete cleanly') + require(not (root_of(record) / 'controller-cleanup-failure.json').exists(), 'controller cleanup failed') + require(sdk.process_identity(result['controller_process']['pid']) is None + and window.job(job_label(record)) is None and window.task_absent(), 'window/controller survivor') + window.service_matches() + require(window.engine() == record['engine'], 'final original engine drift') + verify_original(sdk, record) + sdk.exclusive(root_of(record) / 'window-audit.json', {'outcome': 'pass', 'intent_sha256': window.intent_sha, + 'source_sha256': source_sha, 'controller_absent': True, 'task_absent': True, 'original_engine_unchanged': True}) + + +def owner(sdk, proposal, proposal_sha, args, code, profile_raw): + require(os.isatty(0), 'foreground owner TTY required') + require(sys.platform == 'darwin', 'native macOS owner required') + require(proposal['owner_process'] is None, 'owner epoch must be filled by the native bootstrap') + record = dict(proposal, owner_process=sdk.process_identity(os.getpid())) + source_sha = sha(code.encode()) + validate(sdk, record, source_sha) + require(pathlib.Path(sys.executable).resolve() == pathlib.Path(record['tools']['python']['path']), + 'owner Python differs from the captured launcher') + require(record['started_unix'] <= time.time() < record['started_unix'] + 30, 'stale owner launch') + root = root_of(record) + intent_sha = sdk.exclusive(root / 'window-intent.json', record) + (root / 'docker-config').mkdir(mode=0o700) + sdk.exclusive(root / 'docker-config/config.json', {'auths': {}}) + window = Window(sdk, record, intent_sha, args.sdk_code) + job = {'Label': job_label(record), 'ProgramArguments': [record['tools']['python']['path'], '-I', '-S', '-c', code, + 'watch', '--record', str(root / 'window-intent.json'), '--record-sha256', intent_sha, + '--intent-json', canonical(record).decode(), '--profile-json', profile_raw.decode(), + '--sdk-code', args.sdk_code, '--profile', args.profile], 'RunAtLoad': True, 'KeepAlive': False, + 'LaunchOnlyOnce': True, 'AbandonProcessGroup': False, 'WorkingDirectory': '/', + 'EnvironmentVariables': window.env(), 'StandardOutPath': '/dev/null', 'StandardErrorPath': '/dev/null'} + window.bootstrap_fd(plistlib.dumps(job)) + while time.monotonic() < window.stop_cutoff: + ready_path = root / 'window-ready.json' + if ready_path.exists(): + ready = json.loads(sdk.regular(ready_path)) + effective = window.job(job_label(record)) + require(ready['intent_sha256'] == intent_sha and ready['source_sha256'] == source_sha + and effective and ready['process']['pid'] == effective['pid'] + and sdk.process_identity(effective['pid']) == ready['process'], 'independent controller readiness mismatch') + break + time.sleep(0.1) + else:raise SystemExit('outside-agent controller did not become ready') + sdk.exclusive(root / 'go.json', {'intent_sha256': intent_sha, 'action': 'one-window'}) + print(json.dumps({'window_intent_sha256': intent_sha, 'source_sha256': source_sha, + 'proposal_sha256': proposal_sha, 'owner_process': record['owner_process']}), flush=True) + with selectors.DefaultSelector() as selector: + selector.register(sys.stdin, selectors.EVENT_READ) + while time.monotonic() < window.cleanup_cutoff: + result_path = root / 'window-result.json' + if result_path.exists() and window.job(job_label(record)) is None and \ + sdk.process_identity(json.loads(sdk.regular(result_path))['controller_process']['pid']) is None: + audit(window, source_sha) + print(json.dumps({'operational_window': 'pass', 'sdk_acceptance': 'separate'}), flush=True) + return + for key, _ in selector.select(0.25): + if not os.read(key.fileobj.fileno(), 1024): + raise SystemExit('owner input closed; independent controller retains cleanup custody') + raise SystemExit('owner final-audit cutoff') + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=('owner', 'watch', 'audit')) + parser.add_argument('--record', required=True) + parser.add_argument('--record-sha256', required=True) + parser.add_argument('--sdk-code', required=True) + parser.add_argument('--profile', required=True) + parser.add_argument('--intent-json') + parser.add_argument('--profile-json') + args = parser.parse_args(argv) + require('-I' in sys.orig_argv and '-S' in sys.orig_argv and '-c' in sys.orig_argv, 'captured outer source required') + source_sha = sha(sys.orig_argv[sys.orig_argv.index('-c') + 1].encode()) + sdk = load_sdk(args.sdk_code) + if args.mode == 'watch': + require(args.intent_json is not None and args.profile_json is not None, 'captured watch inputs required') + raw = args.intent_json.encode() + profile_raw = args.profile_json.encode() + else: + require(args.intent_json is None and args.profile_json is None, 'unexpected watch inputs') + raw = sdk.regular(pathlib.Path(args.record), 512 * 1024) + profile_raw = sdk.regular(pathlib.Path(args.profile), 64 * 1024) + require(sha(raw) == args.record_sha256, 'window intent digest mismatch') + record = json.loads(raw) + require(sha(profile_raw) == sdk.PROFILE_SHA, 'wrong frozen SDK profile') + if args.mode == 'owner': + owner(sdk, record, args.record_sha256, args, sys.orig_argv[sys.orig_argv.index('-c') + 1], profile_raw) + return + window = Window(sdk, record, args.record_sha256, args.sdk_code) + if args.mode == 'watch': + require(sys.platform == 'darwin' and os.getppid() == 1, 'outside-agent launchd parent required') + require(record['uid'] == os.getuid() and re.fullmatch('[0-9a-f]{16}', record['token']), 'wrong controller identity') + job = window.job(job_label(record)) + require(job and job['pid'] == os.getpid(), 'controller does not own its launchd registration') + try: + validate(sdk, record, source_sha) + require(pathlib.Path(sys.executable).resolve() == pathlib.Path(record['tools']['python']['path']), + 'controller Python differs from captured launcher') + require(record['started_unix'] <= time.time() < record['started_unix'] + 30, 'stale outer launch') + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + except BaseException: + window.remove_controller() + raise + window.run(source_sha, json.loads(profile_raw)) + else: + validate(sdk, record, source_sha) + audit(window, source_sha) + + +if __name__ == '__main__': + main() From 0d3ca15e7c6ef88e8f23e16dd7ab08ebb4e8a674 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:47:25 +0000 Subject: [PATCH 12/14] test: separate window exception setup from invocation --- scripts/acceptance/colima-window/test_window.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/acceptance/colima-window/test_window.py b/scripts/acceptance/colima-window/test_window.py index 3275208..630edf3 100644 --- a/scripts/acceptance/colima-window/test_window.py +++ b/scripts/acceptance/colima-window/test_window.py @@ -409,11 +409,12 @@ def bootstrap(raw): self.w.job=mock.Mock(return_value={'pid':4321}) args=type('Args',(),{'sdk_code':CODE,'profile':str(self.root/'profile')})() profile=(HERE.parent/'condition-history/guest-profile.json').read_bytes() + code=(HERE/'window.py').read_text() with mock.patch.object(window.os,'isatty',return_value=True), mock.patch.object(window.sys,'platform','darwin'), \ mock.patch.object(window,'validate'), mock.patch.object(window,'Window',return_value=self.w), \ mock.patch.object(window.selectors,'DefaultSelector',side_effect=SystemExit('test after go')), \ mock.patch('builtins.print'), self.assertRaisesRegex(SystemExit,'test after go'): - window.owner(self.sdk,proposal,'a'*64,args,(HERE/'window.py').read_text(),profile) + window.owner(self.sdk,proposal,'a'*64,args,code,profile) job=captured[0] self.assertFalse(job['KeepAlive']);self.assertTrue(job['RunAtLoad'] and job['LaunchOnlyOnce']) self.assertFalse(job['AbandonProcessGroup']) From 3603df00c791a476564fcdf609187b4c48cd1bb9 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:50:20 +0000 Subject: [PATCH 13/14] test: keep failed Colima restoration terminal --- scripts/acceptance/colima-window/test_window.py | 6 +++++- scripts/acceptance/colima-window/window.py | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/acceptance/colima-window/test_window.py b/scripts/acceptance/colima-window/test_window.py index 630edf3..97e1bc5 100644 --- a/scripts/acceptance/colima-window/test_window.py +++ b/scripts/acceptance/colima-window/test_window.py @@ -481,14 +481,18 @@ def test_late_task_start_refuses_before_popen(self): def test_restoration_fault_notifies_without_retry_or_pass(self): call=self.call + attempts=[] def broken(argv,cutoff,**kwargs): - if 'bootstrap' in argv:raise SystemExit('restoration bootstrap failed') + if 'bootstrap' in argv: + attempts.append(argv) + raise SystemExit('restoration bootstrap failed') return call(argv,cutoff,**kwargs) self.w.call=broken result=self.run_fake_task() self.assertEqual(result['outcome'],'fail');self.assertTrue(result['operator_notified']) self.assertFalse(self.loaded);self.assertFalse(self.controller) self.assertEqual(len([x for x in self.calls if 'bootout' in x]),1) + self.assertEqual(len(attempts),1) self.assertFalse(self.bootstraps) def test_original_file_symlink_hardlink_and_writable_input_refuse(self): diff --git a/scripts/acceptance/colima-window/window.py b/scripts/acceptance/colima-window/window.py index 07d95f6..b7cf734 100644 --- a/scripts/acceptance/colima-window/window.py +++ b/scripts/acceptance/colima-window/window.py @@ -368,6 +368,7 @@ def notify_failure(self): def run(self, source_sha, profile): original = None + restoration_attempted = False result = {'outcome': 'fail', 'intent_sha256': self.intent_sha, 'source_sha256': source_sha} try: original = verify_original(self.sdk, self.record) @@ -381,12 +382,16 @@ def run(self, source_sha, profile): self.stop_default() self.start_task(profile) self.monitor_task() + restoration_attempted = True result.update(self.restore(original), outcome='pending_independent_audit') except BaseException as error: result['error_type'] = type(error).__name__ if self.stop_attempted: - try:result.update(self.restore(original)) - except BaseException:result['operator_notified'] = self.notify_failure() + if restoration_attempted: + result['operator_notified'] = self.notify_failure() + else: + try:result.update(self.restore(original)) + except BaseException:result['operator_notified'] = self.notify_failure() finally: try: result['controller_process'] = self.sdk.process_identity(os.getpid()) From bdbb4d451271742c36368a9a0c30cbc65baddb24 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:19:06 +0000 Subject: [PATCH 14/14] test: close Colima absence and fault-evidence authority --- scripts/acceptance/colima-window/README.md | 18 +- .../acceptance/colima-window/test_window.py | 182 +++++++++++++++++- scripts/acceptance/colima-window/window.py | 163 +++++++++++++--- 3 files changed, 329 insertions(+), 34 deletions(-) diff --git a/scripts/acceptance/colima-window/README.md b/scripts/acceptance/colima-window/README.md index 4e91e77..2d65a48 100644 --- a/scripts/acceptance/colima-window/README.md +++ b/scripts/acceptance/colima-window/README.md @@ -27,7 +27,7 @@ contact Docker nor load/unload a service or create a VM. ## Contingent invocation and custody A private proposal supplies the exact source digest, host boot/UID, original -LaunchAgent bytes/effective properties and arguments, six launcher hashes, +LaunchAgent bytes/effective properties and arguments, seven launcher hashes (including `/bin/ps`), original profile/configuration hashes, original disk device/inode identities, local Docker endpoint and idle engine ID/version/architecture/CPU/image/volume inventory. The supported original service invocation is its existing foreground @@ -84,6 +84,22 @@ fallback. Host reboot or killing the independent controller is outside this same-boot guarantee and requires operator recovery; no unconditional boot-time restoration service is installed. +Only exit 113 with empty stdout and the selected native two-line stderr schema +(`Bad request.` followed by the exact queried label and GUI UID) means a service +is absent. Any mixed, additional or mismatched diagnostic refuses restoration +and auditing. Effective-service arguments are read through the separately +hash-bound and reverified `ps` tool. + +Native failures retain the failed stage, tool/operation, exit status or OS errno, +and each captured stdout/stderr byte count and SHA-256. Excerpts contain only a +fixed reviewed list of diagnostic phrases; arbitrary output, arguments, +environment values and credentials are never copied into receipts or terminals. +Each command still captures at most 2 MiB total. The final result retains the +first eight and latest eight faults plus their total count, including any +failed restoration attempt. Owner bootstrap and controller-removal failures +also retain bounded evidence. These private receipts are recovery evidence, +not additional operating authority. + A normal restoration bootstraps the captured original plist, starts that same service and independently reads back its effective arguments/properties, all original file/disk bindings, idle engine fingerprint and inventories. A receipt diff --git a/scripts/acceptance/colima-window/test_window.py b/scripts/acceptance/colima-window/test_window.py index 97e1bc5..0036764 100644 --- a/scripts/acceptance/colima-window/test_window.py +++ b/scripts/acceptance/colima-window/test_window.py @@ -51,12 +51,12 @@ def epoch(pid): self.native_fault, self.bad_properties, self.bad_arguments = None, False, False self.original_files = {} tools = {} - for name in ('python', 'launchctl', 'plutil', 'colima', 'docker', 'dm'): + for name in ('python', 'launchctl', 'plutil', 'colima', 'docker', 'ps', 'dm'): path = self.temp / name path.write_bytes(('fake executable ' + name).encode()) tools[name] = self.bind(path) # Native absolute launchctl/plutil paths have synthetic, checked bytes. - for name, path in [('launchctl', '/bin/launchctl'), ('plutil', '/usr/bin/plutil')]: + for name, path in [('launchctl', '/bin/launchctl'), ('plutil', '/usr/bin/plutil'), ('ps', '/bin/ps')]: tools[name]['path'] = path self.original_files[path] = ('fake executable ' + name).encode() verify = window.verify_file @@ -123,7 +123,8 @@ def call(self, argv, cutoff, *, pass_fds=(), stop=False, check=True): if args[0] == 'print': controller = args[1].endswith(window.job_label(self.record)) if not (self.controller if controller else self.loaded): - return 113, b'', b'Could not find service' + label=args[1].split('/')[-1] + return 113, b'', ('Bad request.\nCould not find service \"'+label+'\" in domain for user gui: '+str(self.record['uid'])+'\n').encode() pid = os.getpid() if controller else 6789 program = self.record['tools']['python' if controller else 'colima']['path'] properties = 'wrong' if self.bad_properties else self.record['service']['properties'] @@ -298,7 +299,7 @@ def test_restoration_inventory_mismatch_is_terminal(self): with mock.patch.object(window.time, 'sleep'), self.assertRaises(SystemExit):self.w.restore(self.original_plist) def test_uncertain_service_query_cannot_authorize_bootstrap(self): - self.w.native = lambda *args, **kwargs:(5,b'',b'permission denied') + self.w.call = lambda *args, **kwargs:(5,b'',b'permission denied') with self.assertRaises(SystemExit):self.w.restore(self.original_plist) self.assertFalse(self.bootstraps) @@ -505,6 +506,179 @@ def test_original_file_symlink_hardlink_and_writable_input_refuse(self): path.chmod(0o600);link.symlink_to(path) with self.assertRaises(SystemExit):window.verify_file(self.sdk,dict(binding,path=str(link))) + def test_permission_error_cannot_be_hidden_in_absence_diagnostic(self): + label=self.record['service']['label'] + error=('permission denied\nBad request.\nCould not find service "'+label+'" in domain for user gui: '+str(self.record['uid'])+'\n').encode() + self.w.call=lambda *args,**kwargs:(113,b'',error) + with self.assertRaises(SystemExit):self.w.job(label) + + def test_only_exact_native_absence_status_and_diagnostics_are_accepted(self): + label=self.record['service']['label'] + exact=('Bad request.\nCould not find service "'+label+'" in domain for user gui: '+str(self.record['uid'])+'\n').encode() + self.w.call=lambda *args,**kwargs:(113,b'',exact) + self.assertIsNone(self.w.job(label)) + self.assertEqual(self.w.native_fault_count,0) + cases=[(1,b'',exact),(5,b'',exact),(113,b'unexpected stdout',exact), + (113,b'',b'permission denied\n'+exact),(113,b'',exact+b'permission denied\n'), + (113,b'',b'Could not find service'),(113,b'',exact.replace(label.encode(),b'other-service')), + (113,b'',exact.replace(str(self.record['uid']).encode(),b'99999')), + (113,b'',exact.replace(b'\n',b'\r\n')),(0,b'',b'permission denied')] + for response in cases: + with self.subTest(response=response): + self.w.call=lambda *args,**kwargs:response + with self.assertRaises(SystemExit):self.w.job(label) + self.assertEqual(self.w.native_fault_count,len(cases)) + + def test_mixed_absence_cannot_bootstrap_restore_or_authorize_final_audit(self): + self.completed() + exact=('Bad request.\nCould not find service "'+window.job_label(self.record)+'" in domain for user gui: '+str(self.record['uid'])+'\n').encode() + self.w.call=lambda *args,**kwargs:(113,b'',b'permission denied\n'+exact) + with self.assertRaises(SystemExit):window.audit(self.w,SOURCE) + with self.assertRaises(SystemExit):self.w.restore(self.original_plist) + self.assertFalse((self.root/'window-audit.json').exists()) + self.assertEqual(self.bootstraps,[self.original_plist]) + + def test_ps_must_be_bound_and_reverified_before_effective_argument_readback(self): + candidate=copy.deepcopy(self.record);candidate['tools'].pop('ps') + with self.assertRaises(SystemExit):window.validate(self.sdk,candidate,SOURCE) + self.original_files['/bin/ps']=b'different native ps' + with self.assertRaises(SystemExit):self.w.service_matches() + self.assertFalse(any(pathlib.Path(call[0]).name=='ps' for call in self.calls)) + self.assertEqual(self.w.native_faults[-1]['tool'],'ps') + + def test_native_status_and_safe_diagnostic_digests_survive_stop_failure(self): + original=self.call + stdout=b'private diagnostic payload' + stderr=b'permission denied token=never-echo-this-value' + def stop_fault(argv,cutoff,**kwargs): + if pathlib.Path(argv[0]).name=='colima':return 77,stdout,stderr + return original(argv,cutoff,**kwargs) + self.w.call=stop_fault + result=self.run_fake_task() + self.assertIn('failed_stage',result) + self.assertEqual(result['failed_stage'],'stop-default') + fault=result['native_faults'][0] + self.assertEqual((fault['tool'],fault['operation'],fault['exit_status']),('colima','stop',77)) + self.assertEqual(fault['stdout']['sha256'],window.sha(stdout)) + self.assertEqual(fault['stderr']['sha256'],window.sha(stderr)) + self.assertEqual(fault['stderr']['safe_excerpt'],['permission denied']) + self.assertNotIn('never-echo-this-value',json.dumps(result)) + self.assertNotIn('private diagnostic payload',json.dumps(result)) + self.assertTrue(result['original_engine_unchanged']) + + def test_failed_recovery_restoration_retains_both_stages_and_diagnostics(self): + original=self.call + def faults(argv,cutoff,**kwargs): + if pathlib.Path(argv[0]).name=='colima':return 77,b'first private failure',b'permission denied' + if 'bootstrap' in argv:return 88,b'restore private failure',b'input/output error' + return original(argv,cutoff,**kwargs) + self.w.call=faults + result=self.run_fake_task() + self.assertIn('failed_stage',result) + self.assertEqual(result['failed_stage'],'stop-default') + self.assertEqual(result['restoration_failure']['failed_stage'],'restore-after-fault') + self.assertEqual([(item['stage'],item['exit_status']) for item in result['native_faults']], + [('stop-default',77),('restore-after-fault',88)]) + self.assertTrue(result['operator_notified']) + + def test_normal_restoration_native_failure_is_terminal_and_recoverable(self): + original=self.call;attempts=[] + def faults(argv,cutoff,**kwargs): + if 'bootstrap' in argv: + attempts.append(argv);return 88,b'',b'input/output error' + return original(argv,cutoff,**kwargs) + self.w.call=faults + result=self.run_fake_task() + self.assertEqual(len(attempts),1) + self.assertEqual(result['failed_stage'],'restore-original') + self.assertEqual(result['native_faults'][-1]['exit_status'],88) + self.assertTrue(result['operator_notified']) + + def test_fault_evidence_has_fixed_size_and_retains_early_and_late_failures(self): + self.w.call=lambda *args,**kwargs:(77,b'private stdout',b'permission denied') + for number in range(40): + self.w.stage='restore-original' + with self.assertRaises(SystemExit):self.w.native('launchctl','kickstart','ignored') + self.w.native_faults[-1]['sequence_for_test']=number + self.assertEqual(self.w.native_fault_count,40) + self.assertEqual(len(self.w.native_faults),16) + self.assertEqual([item['sequence_for_test'] for item in self.w.native_faults],list(range(8))+list(range(32,40))) + self.assertLess(len(window.canonical(self.w.failure_details(SystemExit()))),16000) + + def test_bounded_call_retains_actual_nonzero_status_and_captured_bytes(self): + class FakeNativeChild: + returncode=91 + def __init__(self): + read,write=os.pipe();os.write(write,b'private stdout');os.close(write);self.stdout=os.fdopen(read,'rb') + read,write=os.pipe();os.write(write,b'permission denied private token');os.close(write);self.stderr=os.fdopen(read,'rb') + def poll(self):return self.returncode + def wait(self,timeout):return self.returncode + self.w.call=window.Window.call.__get__(self.w,window.Window) + self.w.stage='stop-default' + with mock.patch.object(window.subprocess,'Popen',return_value=FakeNativeChild()), self.assertRaises(window.NativeFailure): + self.w.native('colima','--profile','default','stop') + fault=self.w.native_faults[-1] + self.assertEqual(fault['exit_status'],91) + self.assertEqual(fault['stdout']['sha256'],window.sha(b'private stdout')) + self.assertEqual(fault['stderr']['sha256'],window.sha(b'permission denied private token')) + self.assertNotIn('private token',json.dumps(fault)) + + def test_owner_bootstrap_failure_retains_private_recovery_evidence(self): + proposal=copy.deepcopy(self.record);proposal['owner_process']=None + proposal['tools']['python']['path']=str(pathlib.Path(sys.executable).resolve()) + (self.root/'go.json').unlink() + args=type('Args',(),{'sdk_code':CODE,'profile':str(self.root/'profile')})() + profile=(HERE.parent/'condition-history/guest-profile.json').read_bytes() + code=(HERE/'window.py').read_text() + self.w.call=lambda *args,**kwargs:(5,b'private bootstrap payload',b'permission denied') + with mock.patch.object(window.os,'isatty',return_value=True), mock.patch.object(window.sys,'platform','darwin'), \ + mock.patch.object(window,'validate'), mock.patch.object(window,'Window',return_value=self.w), \ + self.assertRaises(window.NativeFailure): + window.owner(self.sdk,proposal,'a'*64,args,code,profile) + result=json.loads((self.root/'owner-failure.json').read_bytes()) + self.assertEqual(result['failed_stage'],'owner-bootstrap') + self.assertEqual(result['native_faults'][0]['exit_status'],5) + self.assertEqual(result['native_faults'][0]['tool'],'plutil') + self.assertFalse((self.root/'go.json').exists()) + self.assertNotIn('private bootstrap payload',json.dumps(result)) + + def test_timeout_fault_retains_killed_status_without_a_native_process(self): + class WaitingChild: + returncode=None + def __init__(self): + read,self.writer=os.pipe();self.stdout=os.fdopen(read,'rb') + read,write=os.pipe();os.close(write);self.stderr=os.fdopen(read,'rb') + def poll(self):return self.returncode + def kill(self):self.returncode=-9;os.close(self.writer) + def wait(self,timeout):return self.returncode + self.w.call=window.Window.call.__get__(self.w,window.Window) + self.w.stage='restore-original' + cutoff=time.monotonic()+0.03 + with mock.patch.object(window.subprocess,'Popen',return_value=WaitingChild()), self.assertRaises(window.NativeFailure): + self.w.native('launchctl','kickstart','ignored',cutoff=cutoff) + fault=self.w.native_faults[-1] + self.assertEqual((fault['reason'],fault['exit_status']),('timeout',-9)) + self.assertEqual(fault['stage'],'restore-original') + self.assertEqual(fault['stdout']['bytes']+fault['stderr']['bytes'],0) + + def test_oversize_diagnostic_binds_only_the_bounded_capture(self): + class FakeNativeChild: + returncode=91 + def __init__(self): + read,write=os.pipe();os.write(write,b'x');os.close(write);self.stdout=os.fdopen(read,'rb') + read,write=os.pipe();os.close(write);self.stderr=os.fdopen(read,'rb') + def poll(self):return self.returncode + def wait(self,timeout):return self.returncode + self.w.call=window.Window.call.__get__(self.w,window.Window) + child=FakeNativeChild() + with mock.patch.object(window.subprocess,'Popen',return_value=child), \ + mock.patch.object(window.os,'read',return_value=b'x'*65536), self.assertRaises(window.NativeFailure): + self.w.native('launchctl','print','ignored') + fault=self.w.native_faults[-1] + self.assertEqual(fault['reason'],'output-limit') + self.assertEqual(fault['stdout']['bytes']+fault['stderr']['bytes'],2*1024**2) + self.assertLess(len(window.canonical(fault)),2000) + def test_minimal_environment_excludes_forwarded_credentials_and_default_lima_home(self): with mock.patch.dict(os.environ,{'SSH_AUTH_SOCK':'secret','DOCKER_AUTH_CONFIG':'secret','LIMA_HOME':'foreign'}): env=self.w.env();stop_env=self.w.env(stop=True) diff --git a/scripts/acceptance/colima-window/window.py b/scripts/acceptance/colima-window/window.py index b7cf734..267169a 100644 --- a/scripts/acceptance/colima-window/window.py +++ b/scripts/acceptance/colima-window/window.py @@ -26,6 +26,24 @@ KEYS = {'schema', 'token', 'source_sha256', 'host_boot_id', 'uid', 'scope', 'owner_process', 'started_unix', 'sdk_intent', 'tools', 'service', 'config_files', 'disks', 'docker_host', 'engine', 'profile_home'} +SAFE_DIAGNOSTICS = (b'permission denied', b'operation not permitted', b'input/output error', + b'could not find service', b'bad request', b'connection refused', + b'cannot connect to the docker daemon', b'no such file or directory') + + +class NativeFailure(SystemExit): + def __init__(self, reason, status, output=b'', errors=b'', os_errno=None): + super().__init__(reason) + self.reason, self.status, self.output, self.errors = reason, status, output, errors + self.os_errno = os_errno + + +def diagnostic(data): + # No arbitrary diagnostic bytes enter the receipt or terminal. Only these + # fixed phrases are exposed; the complete bounded captured bytes are hashed. + lower = data.lower() + return {'bytes': len(data), 'sha256': sha(data), + 'safe_excerpt': [phrase.decode() for phrase in SAFE_DIAGNOSTICS if phrase in lower]} def require(ok, message): @@ -134,9 +152,10 @@ def validate(sdk, record, source_sha): require(record['docker_host'].startswith('unix:///') and pathlib.Path(record['profile_home']).is_absolute(), 'local original profile required') home = pathlib.Path(record['profile_home']) require(home.resolve() == home and record['docker_host'] == 'unix://' + str(home / 'default/docker.sock'), 'wrong default engine endpoint') - require(set(record['tools']) == {'python', 'launchctl', 'plutil', 'colima', 'docker', 'dm'}, 'wrong window tool set') + require(set(record['tools']) == {'python', 'launchctl', 'plutil', 'colima', 'docker', 'ps', 'dm'}, 'wrong window tool set') require(record['tools']['launchctl']['path'] == '/bin/launchctl' - and record['tools']['plutil']['path'] == '/usr/bin/plutil', 'native service tools required') + and record['tools']['plutil']['path'] == '/usr/bin/plutil' + and record['tools']['ps']['path'] == '/bin/ps', 'native service tools required') service = record['service'] require(set(service) == {'label', 'plist', 'arguments', 'keep_alive', 'properties', 'vm_processes'} and service['label'] != job_label(record) and not service['label'].startswith('sdk-rust-') @@ -175,6 +194,32 @@ def __init__(self, sdk, record, intent_sha, sdk_code): self.launched = False self.stop_attempted = False self.original_plist = None + self.stage = 'created' + self.native_faults = [] + self.native_fault_count = 0 + self.last_native = None + + def evidence(self, tool, argv, status, output, errors, reason): + operation = argv[2] if tool == 'docker' and len(argv) > 2 else (argv[0] if argv else '') + if tool == 'colima':operation = 'stop' + if tool == 'ps':operation = 'command' + if tool == 'dm':operation = 'notify' + if operation not in {'print', 'bootout', 'bootstrap', 'kickstart', 'remove', '-lint', + 'info', 'ps', 'image', 'volume', 'stop', 'command', 'notify'}:operation = 'other' + return {'stage': self.stage, 'tool': tool, 'operation': operation, 'exit_status': status, + 'reason': reason, 'stdout': diagnostic(output), 'stderr': diagnostic(errors)} + + def remember_fault(self, evidence): + self.native_fault_count += 1 + # Keep the first eight failures and latest eight, including late restore + # or notification faults, within a fixed receipt/storage bound. + if len(self.native_faults) == 16:del self.native_faults[8] + self.native_faults.append(evidence) + + def failure_details(self, error): + return {'failed_stage': self.stage, 'error_type': type(error).__name__, + 'native_fault_count': self.native_fault_count, 'native_faults': list(self.native_faults), + 'last_native_result': self.last_native} def env(self, stop=False): result = {'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'HOME': str(pathlib.Path.home()), @@ -187,7 +232,7 @@ def env(self, stop=False): def call(self, argv, cutoff, *, pass_fds=(), stop=False, check=True): remaining = cutoff - time.monotonic() - require(remaining > 0, 'window command cutoff') + if remaining <= 0:raise NativeFailure('deadline', None) child = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, pass_fds=pass_fds, env=self.env(stop), close_fds=True) output, errors = bytearray(), bytearray() @@ -197,28 +242,62 @@ def call(self, argv, cutoff, *, pass_fds=(), stop=False, check=True): selector.register(child.stdout, selectors.EVENT_READ, output) selector.register(child.stderr, selectors.EVENT_READ, errors) while selector.get_map(): - require(time.monotonic() < end, 'window native command timed out') + if time.monotonic() >= end: + raise NativeFailure('timeout', None, bytes(output), bytes(errors)) for key, _ in selector.select(min(0.1, max(0, end - time.monotonic()))): data = os.read(key.fileobj.fileno(), 65536) if not data:selector.unregister(key.fileobj);continue - require(len(output) + len(errors) + len(data) <= 2 * 1024**2, 'oversize native window response') + if len(output) + len(errors) + len(data) > 2 * 1024**2: + key.data.extend(data[:2 * 1024**2 - len(output) - len(errors)]) + raise NativeFailure('output-limit', None, bytes(output), bytes(errors)) key.data.extend(data) child.wait(timeout=max(0.001, end - time.monotonic())) - require(not check or child.returncode == 0, 'window native control failed') + if check and child.returncode != 0: + raise NativeFailure('exit-status', child.returncode, bytes(output), bytes(errors)) return child.returncode, bytes(output), bytes(errors) + except (SystemExit, OSError, subprocess.TimeoutExpired) as error: + if child.poll() is None:child.kill();child.wait(timeout=2) + if isinstance(error, NativeFailure): + if error.status is None:error.status = child.returncode + raise + raise NativeFailure(type(error).__name__, child.returncode, bytes(output), bytes(errors), + getattr(error, 'errno', None)) from None finally: if child.poll() is None:child.kill();child.wait(timeout=2) child.stdout.close();child.stderr.close() - def native(self, name, *argv, cutoff=None, **kwargs): - verify_file(self.sdk, self.record['tools'][name]) - return self.call([self.record['tools'][name]['path'], *argv], cutoff or self.restore_cutoff, **kwargs) + def native(self, name, *argv, cutoff=None, check=True, **kwargs): + try: + verify_file(self.sdk, self.record['tools'][name]) + result = self.call([self.record['tools'][name]['path'], *argv], + cutoff or self.restore_cutoff, check=False, **kwargs) + except NativeFailure as error: + self.last_native = dict(self.evidence(name, argv, error.status, error.output, error.errors, error.reason), + os_errno=error.os_errno) + self.remember_fault(self.last_native) + raise + except (OSError, SystemExit) as error: + self.last_native = dict(self.evidence(name, argv, None, b'', b'', type(error).__name__), + os_errno=getattr(error, 'errno', None)) + self.remember_fault(self.last_native) + raise + self.last_native = self.evidence(name, argv, *result, 'returned') + if check and result[0] != 0: + self.remember_fault(dict(self.last_native, reason='exit-status')) + raise NativeFailure('exit-status', *result) + return result def job(self, label, cutoff=None): code, output, error = self.native('launchctl', 'print', target(self.record, label), cutoff=cutoff, check=False) if code: - require(b'Could not find service' in error, 'uncertain launchd query') - return None + expected = ('Bad request.\nCould not find service "' + label + + '" in domain for user gui: ' + str(self.record['uid']) + '\n').encode() + if code == 113 and output == b'' and error == expected:return None + self.remember_fault(dict(self.last_native, reason='uncertain-absence')) + raise SystemExit('uncertain launchd query') + if error: + self.remember_fault(dict(self.last_native, reason='unexpected-print-diagnostic')) + raise SystemExit('unexpected launchd diagnostic') text = output.decode() def one(name, optional=False): # Only top-level fields, never a similarly named line inside the @@ -246,7 +325,7 @@ def service_matches(self): job = self.job(self.record['service']['label']) require(job and job['pid'] and pathlib.Path(job['program']).resolve() == pathlib.Path(self.record['tools']['colima']['path']) and job['properties'] == self.record['service']['properties'], 'effective original service changed') - command = self.call(['/bin/ps', '-ww', '-p', str(job['pid']), '-o', 'command='], self.restore_cutoff)[1] + command = self.native('ps', '-ww', '-p', str(job['pid']), '-o', 'command=')[1] require(shlex.split(command.decode()) == self.record['service']['arguments'], 'effective original service arguments changed') return job @@ -361,39 +440,53 @@ def restore(self, original_plist): def notify_failure(self): # The host's existing local dm cache is used only for a real fault. # No credential or vault value is copied into the task guest or a log. + previous, self.stage = self.stage, 'fault-notify' try: self.native('dm', 'sdk-rust: default Colima window has unresolved task/restoration custody; inspect the private window receipt.', cutoff=self.cleanup_cutoff) return True except BaseException:return False + finally:self.stage = previous def run(self, source_sha, profile): original = None restoration_attempted = False result = {'outcome': 'fail', 'intent_sha256': self.intent_sha, 'source_sha256': source_sha} try: + self.stage = 'preflight' original = verify_original(self.sdk, self.record) self.original_plist = original self.sdk.exclusive(self.root / 'window-ready.json', {'intent_sha256': self.intent_sha, 'source_sha256': source_sha, 'process': self.sdk.process_identity(os.getpid())}) + self.stage = 'wait-go' while not (self.root / 'go.json').exists(): require(self.owner_alive() and time.monotonic() < self.stop_cutoff, 'owner/go window lost') time.sleep(0.1) require(json.loads(self.sdk.regular(self.root / 'go.json')) == {'intent_sha256': self.intent_sha, 'action': 'one-window'}, 'wrong window request') + self.stage = 'stop-default' self.stop_default() + self.stage = 'start-task' self.start_task(profile) + self.stage = 'monitor-task' self.monitor_task() restoration_attempted = True + self.stage = 'restore-original' result.update(self.restore(original), outcome='pending_independent_audit') except BaseException as error: - result['error_type'] = type(error).__name__ + result.update(self.failure_details(error)) if self.stop_attempted: if restoration_attempted: result['operator_notified'] = self.notify_failure() else: - try:result.update(self.restore(original)) - except BaseException:result['operator_notified'] = self.notify_failure() + try: + self.stage = 'restore-after-fault' + result.update(self.restore(original)) + except BaseException as restoration_error: + result['restoration_failure'] = self.failure_details(restoration_error) + result['operator_notified'] = self.notify_failure() finally: try: + result.update(native_fault_count=self.native_fault_count, native_faults=list(self.native_faults), + last_native_result=self.last_native) result['controller_process'] = self.sdk.process_identity(os.getpid()) self.sdk.exclusive(self.root / 'window-result.json', result) finally: @@ -402,11 +495,16 @@ def run(self, source_sha, profile): def remove_controller(self): # Receipt precedes self-removal. A separate invocation must prove that # this job/process is gone; this process cannot certify its own absence. + self.stage = 'controller-remove' try: self.native('launchctl', 'remove', job_label(self.record), cutoff=self.cleanup_cutoff) - except BaseException: + except BaseException as error: + detail = self.failure_details(error) + notified = self.notify_failure() + detail.update(native_fault_count=self.native_fault_count, native_faults=list(self.native_faults), + last_native_result=self.last_native) self.sdk.exclusive(self.root / 'controller-cleanup-failure.json', { - 'outcome': 'fail', 'intent_sha256': self.intent_sha, 'operator_notified': self.notify_failure()}) + **detail, 'outcome': 'fail', 'intent_sha256': self.intent_sha, 'operator_notified': notified}) raise @@ -447,18 +545,25 @@ def owner(sdk, proposal, proposal_sha, args, code, profile_raw): '--sdk-code', args.sdk_code, '--profile', args.profile], 'RunAtLoad': True, 'KeepAlive': False, 'LaunchOnlyOnce': True, 'AbandonProcessGroup': False, 'WorkingDirectory': '/', 'EnvironmentVariables': window.env(), 'StandardOutPath': '/dev/null', 'StandardErrorPath': '/dev/null'} - window.bootstrap_fd(plistlib.dumps(job)) - while time.monotonic() < window.stop_cutoff: - ready_path = root / 'window-ready.json' - if ready_path.exists(): - ready = json.loads(sdk.regular(ready_path)) - effective = window.job(job_label(record)) - require(ready['intent_sha256'] == intent_sha and ready['source_sha256'] == source_sha - and effective and ready['process']['pid'] == effective['pid'] - and sdk.process_identity(effective['pid']) == ready['process'], 'independent controller readiness mismatch') - break - time.sleep(0.1) - else:raise SystemExit('outside-agent controller did not become ready') + try: + window.stage = 'owner-bootstrap' + window.bootstrap_fd(plistlib.dumps(job)) + window.stage = 'owner-readiness' + while time.monotonic() < window.stop_cutoff: + ready_path = root / 'window-ready.json' + if ready_path.exists(): + ready = json.loads(sdk.regular(ready_path)) + effective = window.job(job_label(record)) + require(ready['intent_sha256'] == intent_sha and ready['source_sha256'] == source_sha + and effective and ready['process']['pid'] == effective['pid'] + and sdk.process_identity(effective['pid']) == ready['process'], 'independent controller readiness mismatch') + break + time.sleep(0.1) + else:raise SystemExit('outside-agent controller did not become ready') + except BaseException as error: + sdk.exclusive(root / 'owner-failure.json', {'intent_sha256': intent_sha, + 'source_sha256': source_sha, **window.failure_details(error)}) + raise sdk.exclusive(root / 'go.json', {'intent_sha256': intent_sha, 'action': 'one-window'}) print(json.dumps({'window_intent_sha256': intent_sha, 'source_sha256': source_sha, 'proposal_sha256': proposal_sha, 'owner_process': record['owner_process']}), flush=True)