From 5057943483ed964bc25638e23c136cfb51517447 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 18:12:47 -0400 Subject: [PATCH 01/16] Add resumable offline delivery replay --- README.md | 12 + RESTORE.md | 14 + ci/required-files.txt | 5 + delivery/v1/replay.py | 369 +++++++++++++++++++++++++++ scripts/test/delivery-replay.test.sh | 189 ++++++++++++++ work/delivery-loop-first/plan.md | 23 ++ 6 files changed, 612 insertions(+) create mode 100755 delivery/v1/replay.py create mode 100755 scripts/test/delivery-replay.test.sh create mode 100644 work/delivery-loop-first/plan.md diff --git a/README.md b/README.md index 5a7015f..bdf5410 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,18 @@ mismatches remain valid only when all evidence is non-passing. The stable wrappe inactive resolver, Control closure, and scanner select it together. The prior generation remains immutable and restorable. +## Inactive offline delivery replay + +`delivery/v1/replay.py` replays one already-supplied local materialization input +through the existing local Git materializer. It then checks one repo-relative +candidate blob against a supplied SHA-256 and records a private, resumable state. +It never executes candidate code or a user command string. + +Review and publisher records are supplied offline test observations. They bind the +exact request and candidate but do not authenticate an actor or authorize a real +publication. Missing review stays waiting; a completed receipt is explicitly an +offline simulation with no authority or qualification. + ## Inactive fake adapter contract matrix `adapter-tests/v1/` runs a fixed 2×2 producer/forge matrix against one unrelated diff --git a/RESTORE.md b/RESTORE.md index dce165d..6326d8a 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -55,6 +55,20 @@ with the restored `scripts/core-contract.sh`. No compiled helper is installed or restored. A future activation must separately qualify and bind a production trusted parent; restoring these files does not select a live profile. +### Restore the inactive offline delivery replay + +Restore the three paths listed under “Inactive offline delivery replay” in +[`ci/required-files.txt`](ci/required-files.txt), then run: + +```sh +bash scripts/test/delivery-replay.test.sh +``` + +The replay is a local offline simulation. It materializes only a caller-owned +candidate, reads one fixed candidate blob, and records test observations. It does +not execute candidate code, select a profile, authenticate review, publish, merge, +deploy, or contact a provider or target. + --- ## 1. Recreate yshifu (the manager) diff --git a/ci/required-files.txt b/ci/required-files.txt index c7dac9b..c53601a 100644 --- a/ci/required-files.txt +++ b/ci/required-files.txt @@ -257,6 +257,11 @@ adapters/local-git-materializer/v1/materialize.sh adapters/local-git-materializer/v1/object-closure.c scripts/test/local-git-materializer-adapter.test.sh +# Inactive offline delivery replay +delivery/v1/replay.py +scripts/test/delivery-replay.test.sh +work/delivery-loop-first/plan.md + # Inactive Claude Code producer normalizer payload adapters/claude-code-producer/v1/normalize.jq scripts/test/default-claude-code-producer-adapter.test.sh diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py new file mode 100755 index 0000000..0ff5a28 --- /dev/null +++ b/delivery/v1/replay.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +"""Run one inactive, offline delivery replay without executing candidate code.""" + +import argparse +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +import tempfile + + +MAX_INPUT_BYTES = 8 * 1024 * 1024 +MAX_OBSERVATION_BYTES = 64 * 1024 +MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 +OID = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}\Z") +ACTOR = re.compile(r"[a-z0-9][a-z0-9._:-]{0,127}\Z") + + +class ReplayError(Exception): + pass + + +def digest_bytes(value): + return hashlib.sha256(value).hexdigest() + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + + +def read_json(path, limit): + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + chunks = [] + remaining = limit + 1 + while remaining: + chunk = os.read(descriptor, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + finally: + os.close(descriptor) + if len(data) > limit: + raise ReplayError("input exceeds its size limit") + try: + return json.loads(data), digest_bytes(data) + except json.JSONDecodeError as error: + raise ReplayError("input is not JSON") from error + + +def private_directory(path): + value = Path(path) + stat = value.stat() + if value.is_symlink() or not value.is_dir() or stat.st_uid != os.getuid(): + raise ReplayError("state directory is not a caller-owned directory") + if stat.st_mode & 0o077: + raise ReplayError("state directory is not private") + return value.resolve() + + +def trusted_file(path): + value = Path(path) + stat = value.stat() + if value.is_symlink() or not value.is_file() or stat.st_size > MAX_INPUT_BYTES: + raise ReplayError("trusted tool is unavailable") + return value.resolve() + + +def disjoint(*paths): + resolved = [Path(path).resolve() for path in paths] + for index, left in enumerate(resolved): + for right in resolved[index + 1:]: + if left == right or left in right.parents or right in left.parents: + raise ReplayError("caller-owned boundaries overlap") + + +def atomic_json(path, value): + encoded = canonical(value) + b"\n" + descriptor, temporary = tempfile.mkstemp(prefix=".replay-", dir=path.parent) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def load_state(path): + if not path.exists(): + return None + value, _ = read_json(path, MAX_OBSERVATION_BYTES) + if not isinstance(value, dict): + raise ReplayError("state journal is malformed") + return value + + +def safe_path(value): + if not isinstance(value, str) or not value or len(value) > 4096: + raise ReplayError("verification path is invalid") + parts = value.split("/") + if any(part in {"", ".", "..", ".git"} or part.endswith((".", " ")) for part in parts): + raise ReplayError("verification path is invalid") + if any("\\" in part or any(ord(char) < 32 for char in part) for part in parts): + raise ReplayError("verification path is invalid") + return value + + +def input_identity(input_value, input_sha, arguments, materializer): + try: + request = input_value["stage_request"] + request_sha = request["sha256"] + body = request["content"]["body"] + source = body["target_revision"]["value"] + source_tree_id = next( + item["value"]["value"]["value"]["object_id"] + for item in body["inputs"] + if item["input_id"] == body["operation"]["arguments"]["source_tree_input_id"] + ) + except (KeyError, StopIteration, TypeError) as error: + raise ReplayError("materialization input lacks an exact source identity") from error + if not isinstance(request_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", request_sha): + raise ReplayError("materialization input request identity is invalid") + if not isinstance(source_tree_id, str) or not OID.fullmatch(source_tree_id): + raise ReplayError("materialization input tree identity is invalid") + if not isinstance(source, dict) or not OID.fullmatch(str(source.get("commit_id", ""))): + raise ReplayError("materialization input commit identity is invalid") + expected = arguments.expected_sha256 + if not re.fullmatch(r"[0-9a-f]{64}", expected): + raise ReplayError("expected verifier digest is invalid") + closure = trusted_file(arguments.closure_helper) + jq_bin = trusted_file(arguments.jq_bin) + materializer_sha = digest_bytes(materializer.read_bytes()) + identity = { + "input_sha256": input_sha, + "request_sha256": request_sha, + "source_commit_id": source["commit_id"], + "source_tree_id": source_tree_id, + "verifier": { + "id": "delivery.fixed-content-sha256.v1", + "path": safe_path(arguments.verify_path), + "expected_sha256": expected, + }, + "materializer_sha256": materializer_sha, + "closure_helper_sha256": digest_bytes(closure.read_bytes()), + "jq_sha256": digest_bytes(jq_bin.read_bytes()), + "source_repository_id": arguments.source_repository_id, + } + identity["run_key"] = digest_bytes(canonical(identity)) + return identity + + +def run_materializer(arguments, materializer): + command = [ + str(materializer), "materialize", str(Path(arguments.input).resolve()), + arguments.source_repository_id, str(Path(arguments.source_git_dir).resolve()), + str(Path(arguments.candidate_root).resolve()), str(Path(arguments.scratch_root).resolve()), + str(Path(arguments.closure_helper).resolve()), str(Path(arguments.jq_bin).resolve()), + ] + environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"} + result = subprocess.run(command, env=environment, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=False) + if result.returncode != 0 or len(result.stdout) > MAX_INPUT_BYTES: + raise ReplayError("materialization did not complete") + try: + response = json.loads(result.stdout) + receipt_text = response["payloads"][0]["data"] + receipt = json.loads(receipt_text) + candidate = receipt["candidate"] + return { + "response_sha256": digest_bytes(result.stdout), + "receipt_sha256": response["payloads"][0]["sha256"], + "candidate_commit_id": candidate["commit_id"], + "candidate_tree_id": candidate["tree_id"], + } + except (KeyError, IndexError, TypeError, json.JSONDecodeError) as error: + raise ReplayError("materializer response is malformed") from error + + +def verify_candidate(candidate_root, candidate_tree, path, expected): + repository = Path(candidate_root).resolve() / "repository.git" + if not repository.is_dir() or repository.is_symlink() or not OID.fullmatch(candidate_tree): + raise ReplayError("candidate repository identity is unavailable") + environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} + object_name = f"{candidate_tree}:{path}" + size = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "cat-file", "-s", object_name], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + if size.returncode != 0 or not size.stdout.strip().isdigit() or int(size.stdout) > MAX_VERIFIED_BLOB_BYTES: + raise ReplayError("fixed verifier cannot read the candidate blob") + blob = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "cat-file", "blob", object_name], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + if blob.returncode != 0 or len(blob.stdout) != int(size.stdout): + raise ReplayError("fixed verifier could not read the candidate blob") + actual = digest_bytes(blob.stdout) + if actual != expected: + raise ReplayError("fixed verifier digest mismatch") + return actual + + +def observation(path, kind, identity, field): + if path is None: + return None + value, source_sha = read_json(path, MAX_OBSERVATION_BYTES) + if not isinstance(value, dict) or value.get("schema_version") != 1 or value.get("kind") != kind: + raise ReplayError("offline observation is malformed") + if not ACTOR.fullmatch(str(value.get("actor_id", ""))): + raise ReplayError("offline observation actor is invalid") + if value.get("request_sha256") != identity["request_sha256"] or value.get("candidate_tree_id") != identity["candidate_tree_id"]: + raise ReplayError("offline observation does not match this candidate") + return {"actor_id": value["actor_id"], field: value.get(field), "sha256": source_sha} + + +def result(state): + print(json.dumps({"kind": "delivery_replay_receipt", "authority": "none", + "qualification": "unavailable", "offline_simulation": True, + "state": state}, sort_keys=True, separators=(",", ":"))) + + +def replay(arguments): + repository = Path(__file__).resolve().parents[2] + materializer = trusted_file(repository / "adapters/local-git-materializer/v1/materialize.sh") + input_value, input_sha = read_json(arguments.input, MAX_INPUT_BYTES) + identity = input_identity(input_value, input_sha, arguments, materializer) + state_dir = private_directory(arguments.state_dir) + disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) + state_path = state_dir / "run.json" + lock_path = state_dir / "replay.lock" + interrupted = {"value": False} + previous_term = signal.getsignal(signal.SIGTERM) + previous_int = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGTERM, lambda *_: interrupted.__setitem__("value", True)) + signal.signal(signal.SIGINT, lambda *_: interrupted.__setitem__("value", True)) + try: + lock_descriptor = os.open(lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(lock_descriptor, "a+b") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + state = load_state(state_path) + if state is not None and state.get("identity", {}).get("run_key") != identity["run_key"]: + result({"phase": "stale", "reason": "run identity changed"}) + return 2 + if state is None: + state = {"schema_version": 1, "kind": "delivery_replay_state", "identity": identity, + "phase": "materializing", "authority": "none", "qualification": "unavailable"} + atomic_json(state_path, state) + if state["phase"] == "failed": + if state.get("recoverable"): + state["recovery"] = "start a new replay with fresh empty candidate, scratch, and state directories" + atomic_json(state_path, state) + result(state) + return 1 + if state["phase"] == "completed-offline": + verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], + identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) + for supplied, kind, field, recorded in ( + (arguments.review_observation, "delivery_replay_review_observation", "verdict", state.get("review")), + (arguments.publisher_observation, "delivery_replay_publisher_observation", "disposition", state.get("publisher")), + ): + if supplied is not None and observation(supplied, kind, state["identity"], field) != recorded: + raise ReplayError("supplied offline observation changed after completion") + result(state) + return 0 + if state["phase"] == "materializing": + try: + state["materialization"] = run_materializer(arguments, materializer) + except ReplayError as error: + state.update({"phase": "failed", "recoverable": True, "reason": str(error)}) + atomic_json(state_path, state) + result(state) + return 1 + state["identity"].update({ + "candidate_commit_id": state["materialization"]["candidate_commit_id"], + "candidate_tree_id": state["materialization"]["candidate_tree_id"], + }) + state["phase"] = "verifying" + atomic_json(state_path, state) + if interrupted["value"]: + result(state) + return 75 + if state["phase"] == "verifying": + try: + state["verification"] = {"id": identity["verifier"]["id"], "path": identity["verifier"]["path"], + "sha256": verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], + identity["verifier"]["path"], identity["verifier"]["expected_sha256"])} + except ReplayError as error: + state.update({"phase": "failed", "recoverable": False, "reason": str(error)}) + atomic_json(state_path, state) + result(state) + return 1 + state["phase"] = "review-wait" + atomic_json(state_path, state) + if interrupted["value"]: + result(state) + return 75 + if state["phase"] == "review-wait": + verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], + identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) + review = observation(arguments.review_observation, "delivery_replay_review_observation", state["identity"], "verdict") + if review is None: + result(state) + return 0 + if review["verdict"] != "clean": + state.update({"phase": "failed", "recoverable": False, "reason": "offline review did not report clean"}) + atomic_json(state_path, state) + result(state) + return 1 + state["review"] = review + state["phase"] = "publish-wait" + atomic_json(state_path, state) + if state["phase"] == "publish-wait": + publisher = observation(arguments.publisher_observation, "delivery_replay_publisher_observation", state["identity"], "disposition") + if publisher is None: + result(state) + return 0 + if publisher["disposition"] != "offline-simulated": + state.update({"phase": "failed", "recoverable": False, "reason": "offline publisher disposition is invalid"}) + atomic_json(state_path, state) + result(state) + return 1 + state["publisher"] = publisher + state["phase"] = "completed-offline" + atomic_json(state_path, state) + result(state) + return 0 + result(state) + return 1 + finally: + signal.signal(signal.SIGTERM, previous_term) + signal.signal(signal.SIGINT, previous_int) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True) + parser.add_argument("--source-repository-id", required=True) + parser.add_argument("--source-git-dir", required=True) + parser.add_argument("--candidate-root", required=True) + parser.add_argument("--scratch-root", required=True) + parser.add_argument("--state-dir", required=True) + parser.add_argument("--closure-helper", required=True) + parser.add_argument("--jq-bin", required=True) + parser.add_argument("--verify-path", required=True) + parser.add_argument("--expected-sha256", required=True) + parser.add_argument("--review-observation") + parser.add_argument("--publisher-observation") + try: + return replay(parser.parse_args()) + except (OSError, ReplayError) as error: + print(f"delivery replay: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh new file mode 100755 index 0000000..3f1c147 --- /dev/null +++ b/scripts/test/delivery-replay.test.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C +export PYTHONDONTWRITEBYTECODE=1 +umask 077 + +root=$(CDPATH='' cd -P -- "${BASH_SOURCE[0]%/*}/../.." && pwd -P) +replay="$root/delivery/v1/replay.py" +fixture_builder="$root/scripts/test/local-git-materializer-fixtures.sh" +closure_source="$root/adapters/local-git-materializer/v1/object-closure.c" +tmp=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-delivery-replay.XXXXXX") +cleanup() { /bin/rm -rf -- "$tmp"; } +trap cleanup EXIT + +sha_file() { /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}'; } +fail() { printf 'FAIL: %s\n' "$1" >&2; exit 1; } +passed=0 +pass() { passed=$((passed + 1)); printf 'ok %s - %s\n' "$passed" "$1"; } + +platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m) +case "$platform" in + Linux:x86_64) asset=jq-linux64; asset_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 ;; + Darwin:x86_64|Darwin:arm64) asset=jq-osx-amd64; asset_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;; + *) fail "unsupported host $platform" ;; +esac +jq_bin="${TMPDIR:-/tmp}/ystack-portable-core-jq16/$asset" +[ -f "$jq_bin" ] && [ ! -L "$jq_bin" ] && [ "$(sha_file "$jq_bin")" = "$asset_sha" ] || + fail 'pinned jq 1.6 is required' + +runtime="$tmp/runtime" +/bin/mkdir -m 700 "$runtime" "$tmp/home" +if [ "$platform" = Darwin:arm64 ]; then + printf '%s\n' '#!/bin/sh' "exec /usr/bin/arch -x86_64 '$jq_bin' \"\$@\"" > "$runtime/jq" +else + /bin/cp "$jq_bin" "$runtime/jq" +fi +/bin/chmod 0555 "$runtime/jq" +jq_bin="$runtime/jq" +/usr/bin/cc -std=c11 -Wall -Wextra -Werror -O2 "$closure_source" -o "$runtime/object-closure" +/bin/chmod 0555 "$runtime/object-closure" + +git_clean() { + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_NO_LAZY_FETCH=1 GIT_TERMINAL_PROMPT=0 /usr/bin/git --no-replace-objects "$@" +} +make_source() { + local destination=$1 blob tree commit + /bin/mkdir -m 700 "$destination" + git_clean init -q --bare "$destination" + blob=$(printf '%s\n' alpha beta | git_clean --git-dir="$destination" hash-object -w --stdin) + tree=$(printf '100644 blob %s\tsource.txt\n' "$blob" | git_clean --git-dir="$destination" mktree) + commit=$(printf '%s\n' source | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_AUTHOR_NAME=fixture \ + GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ + GIT_COMMITTER_EMAIL=fixture@example.invalid GIT_AUTHOR_DATE=2000-01-01T00:00:00Z \ + GIT_COMMITTER_DATE=2000-01-01T00:00:00Z /usr/bin/git --git-dir="$destination" commit-tree "$tree") + git_clean --git-dir="$destination" update-ref refs/heads/main "$commit" + printf '%s %s\n' "$commit" "$tree" +} + +read -r source_commit source_tree < <(make_source "$tmp/source.git") +"$fixture_builder" build "$tmp/fixture" "$jq_bin" sha1 "$source_commit" "$source_tree" +base_input="$tmp/fixture/input.json" +expected_changed=$(printf '%s\n' alpha beta gamma | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}') + +run_replay() { + local name=$1 input=$2 expected=$3 + local state="$tmp/$name-state" candidate="$tmp/$name-candidate" scratch="$tmp/$name-scratch" + /bin/mkdir -m 700 "$state" "$candidate" "$scratch" + python3 "$replay" --input "$input" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$candidate" --scratch-root "$scratch" \ + --state-dir "$state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected" +} + +run_replay changed "$base_input" "$expected_changed" >"$tmp/changed.out" +jq -e '.state.phase=="review-wait" and .authority=="none" and .offline_simulation==true' "$tmp/changed.out" >/dev/null || + fail missing-review-waits +request_sha=$(jq -r '.identity.request_sha256' "$tmp/changed-state/run.json") +candidate_tree=$(jq -r '.identity.candidate_tree_id' "$tmp/changed-state/run.json") +pass 'changed materialization and fixed read-only verification wait for review' + +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" +jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/review.json" --publisher-observation "$tmp/publisher.json" >"$tmp/completed.out" +jq -e '.state.phase=="completed-offline" and .state.publisher.disposition=="offline-simulated"' "$tmp/completed.out" >/dev/null || + fail completed-offline +cp "$tmp/completed.out" "$tmp/completed-first.out" +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/completed-repeat.out" +cmp "$tmp/completed-first.out" "$tmp/completed-repeat.out" || fail duplicate-completion-output +pass 'offline review and publisher observations complete once and replay deterministically' + +if run_replay verifier-failure "$base_input" "$(printf '0%.0s' {1..64})" >"$tmp/verifier-failure.out" 2>&1; then + fail fixed-verifier-failure +fi +jq -e '.state.phase=="failed" and (.state.reason|contains("digest mismatch"))' "$tmp/verifier-failure.out" >/dev/null || + fail fixed-verifier-failure-state +pass 'fixed verifier failure is terminal and explicit' + +mkdir -m 700 "$tmp/mismatch-state" "$tmp/mismatch-candidate" "$tmp/mismatch-scratch" +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/mismatch-candidate" --scratch-root "$tmp/mismatch-scratch" --state-dir "$tmp/mismatch-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" > /dev/null +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$(printf '0%.0s' {1..40})"'","verdict":"clean"}' >"$tmp/mismatch-review.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/mismatch-candidate" --scratch-root "$tmp/mismatch-scratch" --state-dir "$tmp/mismatch-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/mismatch-review.json" >"$tmp/mismatch.out" 2>&1; then fail mismatched-review; fi +grep -Fq 'does not match this candidate' "$tmp/mismatch.out" || fail mismatched-review-error +pass 'mismatched supplied review cannot complete the replay' + +empty_input="$tmp/empty-input.json" +jq -S -c '(.stage_request.content.body.inputs[] | select(.input_id=="input.producer-patch") | .value.value.value.sha256) = $sha | + (.payloads[] | select(.input_id=="input.producer-patch") | .data) = "" | + (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .content.data) = "" | + (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .sha256) = $sha' \ + --arg sha "$(printf '' | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}')" "$base_input" >"$empty_input" +empty_request="$tmp/empty-request.json" +jq -S -c '.stage_request.content' "$empty_input" >"$empty_request" +empty_request_sha=$(sha_file "$empty_request") +jq -S -c --arg sha "$empty_request_sha" '.stage_request.sha256=$sha' "$empty_input" >"$tmp/empty-final.json" +source_digest=$(printf '%s\n' alpha beta | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}') +run_replay no-change "$tmp/empty-final.json" "$source_digest" >"$tmp/no-change.out" +jq -e '.state.phase=="review-wait" and .state.materialization.candidate_tree_id==.state.identity.source_tree_id' "$tmp/no-change.out" >/dev/null || + fail no-change +pass 'empty producer patch records a no-change candidate before review' + +mkdir -m 700 "$tmp/interrupted-state" "$tmp/interrupted-candidate" "$tmp/interrupted-scratch" +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/interrupted-candidate" --scratch-root "$tmp/interrupted-scratch" --state-dir "$tmp/interrupted-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/interrupted.out" & +interrupted_pid=$! +interrupted_wait=0 +while [ ! -f "$tmp/interrupted-state/run.json" ]; do + if ! kill -0 "$interrupted_pid" 2>/dev/null; then + wait "$interrupted_pid" || : + sed -n '1,12p' "$tmp/interrupted.out" >&2 + fail interrupted-start + fi + interrupted_wait=$((interrupted_wait + 1)) + if [ "$interrupted_wait" -gt 100 ]; then + kill -TERM "$interrupted_pid" 2>/dev/null || : + wait "$interrupted_pid" || : + sed -n '1,12p' "$tmp/interrupted.out" >&2 + fail interrupted-start-timeout + fi + sleep 0.1 +done +kill -TERM "$interrupted_pid" +if wait "$interrupted_pid"; then fail interrupted-run; fi +[ "$(jq -r '.phase' "$tmp/interrupted-state/run.json")" = verifying ] || fail interrupted-state +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/interrupted-candidate" --scratch-root "$tmp/interrupted-scratch" --state-dir "$tmp/interrupted-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/interrupted-retry.out" +jq -e '.state.phase=="review-wait"' "$tmp/interrupted-retry.out" >/dev/null || fail interrupted-retry +pass 'interruption after materialization resumes without a duplicate candidate output' + +mkdir -m 700 "$tmp/stale-state" "$tmp/stale-candidate" "$tmp/stale-scratch" +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/stale-candidate" --scratch-root "$tmp/stale-scratch" --state-dir "$tmp/stale-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" > /dev/null +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/stale-candidate" --scratch-root "$tmp/stale-scratch" --state-dir "$tmp/stale-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$source_digest" >"$tmp/stale.out"; then fail changed-input-stale; fi +jq -e '.state.phase=="stale"' "$tmp/stale.out" >/dev/null || fail changed-input-stale-state +pass 'changed verifier input cannot reuse the prior run' +changed_input="$tmp/changed-input.json" +jq -S -c '.attempt.attempt_id="attempt.changed"' "$base_input" >"$changed_input" +if python3 "$replay" --input "$changed_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/stale-candidate" --scratch-root "$tmp/stale-scratch" --state-dir "$tmp/stale-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/changed-input-stale.out"; then fail changed-materialization-input-stale; fi +jq -e '.state.phase=="stale"' "$tmp/changed-input-stale.out" >/dev/null || fail changed-materialization-input-stale-state +pass 'changed materialization input cannot reuse the prior run' + +printf 'delivery replay: %s focused checks passed\n' "$passed" diff --git a/work/delivery-loop-first/plan.md b/work/delivery-loop-first/plan.md new file mode 100644 index 0000000..0200d0d --- /dev/null +++ b/work/delivery-loop-first/plan.md @@ -0,0 +1,23 @@ +# Delivery loop first plan + +This PR adds one inactive offline replay slice. It owns only these paths: + +- `delivery/v1/replay.py` +- `scripts/test/delivery-replay.test.sh` and its test-owned fixtures +- `work/delivery-loop-first/plan.md` +- `README.md`, `RESTORE.md`, and `ci/required-files.txt` + +The replay accepts an existing canonical local-materialization input and +caller-owned source, candidate, scratch, and private state directories. It calls +the existing local materializer, verifies one repo-relative candidate blob against +one supplied SHA-256, and journals identities and phase atomically. It never runs +candidate code or a user command string. + +States are `materializing`, `verifying`, `review-wait`, `publish-wait`, `failed`, +and `completed-offline`. Review and publisher records are supplied offline test +observations. They name the exact request and candidate plus an actor, but do not +authenticate anyone or authorize publication. Missing review remains waiting. + +The slice is not profile selection, qualification, model execution, target access, +deployment, release, install, merge, or production publication. It is not the +whole delivery loop. From 0aca16fc7dedd145c7e2ca432364046de251a77c Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 18:15:11 -0400 Subject: [PATCH 02/16] Simplify delivery replay state loading --- delivery/v1/replay.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 0ff5a28..bda8568 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -100,15 +100,6 @@ def atomic_json(path, value): os.unlink(temporary) -def load_state(path): - if not path.exists(): - return None - value, _ = read_json(path, MAX_OBSERVATION_BYTES) - if not isinstance(value, dict): - raise ReplayError("state journal is malformed") - return value - - def safe_path(value): if not isinstance(value, str) or not value or len(value) > 4096: raise ReplayError("verification path is invalid") @@ -250,7 +241,11 @@ def replay(arguments): lock_descriptor = os.open(lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) with os.fdopen(lock_descriptor, "a+b") as lock: fcntl.flock(lock, fcntl.LOCK_EX) - state = load_state(state_path) + state = None + if state_path.exists(): + state, _ = read_json(state_path, MAX_OBSERVATION_BYTES) + if not isinstance(state, dict): + raise ReplayError("state journal is malformed") if state is not None and state.get("identity", {}).get("run_key") != identity["run_key"]: result({"phase": "stale", "reason": "run identity changed"}) return 2 From 90ee42f019785b10f06a1f6aff4f1222b38c7c73 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 18:21:31 -0400 Subject: [PATCH 03/16] Revalidate publish wait replay state --- delivery/v1/replay.py | 7 +++++++ scripts/test/delivery-replay.test.sh | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index bda8568..4fe0e4b 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -318,6 +318,13 @@ def replay(arguments): state["phase"] = "publish-wait" atomic_json(state_path, state) if state["phase"] == "publish-wait": + verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], + identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) + if arguments.review_observation is not None and ( + observation(arguments.review_observation, "delivery_replay_review_observation", + state["identity"], "verdict") != state.get("review") + ): + raise ReplayError("supplied offline review changed after review wait") publisher = observation(arguments.publisher_observation, "delivery_replay_publisher_observation", state["identity"], "disposition") if publisher is None: result(state) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 3f1c147..46de5f8 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -88,6 +88,13 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits +jq -S -c '.note="changed after review wait"' "$tmp/review.json" >"$tmp/changed-review.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/changed-review.json" --publisher-observation "$tmp/publisher.json" >"$tmp/changed-review.out" 2>&1; then fail changed-review-after-wait; fi +grep -Fq 'review changed after review wait' "$tmp/changed-review.out" || fail changed-review-after-wait-error +pass 'a changed supplied review cannot advance publish wait' python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ From 653771542d68df825c2a21417bb1c9f5599e05f3 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 18:31:36 -0400 Subject: [PATCH 04/16] Snapshot delivery replay input --- delivery/v1/replay.py | 52 +++++++++++++++++++++++----- scripts/test/delivery-replay.test.sh | 29 ++++++++++++++++ 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 4fe0e4b..a241f82 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -33,7 +33,7 @@ def canonical(value): return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() -def read_json(path, limit): +def read_bytes(path, limit): descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) try: chunks = [] @@ -49,12 +49,21 @@ def read_json(path, limit): os.close(descriptor) if len(data) > limit: raise ReplayError("input exceeds its size limit") + return data + + +def parse_json(data): try: - return json.loads(data), digest_bytes(data) + return json.loads(data) except json.JSONDecodeError as error: raise ReplayError("input is not JSON") from error +def read_json(path, limit): + data = read_bytes(path, limit) + return parse_json(data), digest_bytes(data) + + def private_directory(path): value = Path(path) stat = value.stat() @@ -82,7 +91,10 @@ def disjoint(*paths): def atomic_json(path, value): - encoded = canonical(value) + b"\n" + atomic_bytes(path, canonical(value) + b"\n") + + +def atomic_bytes(path, encoded): descriptor, temporary = tempfile.mkstemp(prefix=".replay-", dir=path.parent) try: with os.fdopen(descriptor, "wb") as handle: @@ -141,6 +153,7 @@ def input_identity(input_value, input_sha, arguments, materializer): "request_sha256": request_sha, "source_commit_id": source["commit_id"], "source_tree_id": source_tree_id, + "source_hash_algorithm": source.get("hash_algorithm"), "verifier": { "id": "delivery.fixed-content-sha256.v1", "path": safe_path(arguments.verify_path), @@ -155,9 +168,9 @@ def input_identity(input_value, input_sha, arguments, materializer): return identity -def run_materializer(arguments, materializer): +def run_materializer(arguments, materializer, input_path, identity): command = [ - str(materializer), "materialize", str(Path(arguments.input).resolve()), + str(materializer), "materialize", str(input_path), arguments.source_repository_id, str(Path(arguments.source_git_dir).resolve()), str(Path(arguments.candidate_root).resolve()), str(Path(arguments.scratch_root).resolve()), str(Path(arguments.closure_helper).resolve()), str(Path(arguments.jq_bin).resolve()), @@ -172,9 +185,22 @@ def run_materializer(arguments, materializer): receipt_text = response["payloads"][0]["data"] receipt = json.loads(receipt_text) candidate = receipt["candidate"] + receipt_sha = response["payloads"][0]["sha256"] + if ( + receipt_sha != digest_bytes(receipt_text.encode()) or + receipt["request_ref"]["sha256"] != identity["request_sha256"] or + response["stage_result"]["body"]["request_ref"]["sha256"] != identity["request_sha256"] or + receipt["source"] != { + "repository_id": identity["source_repository_id"], + "hash_algorithm": identity["source_hash_algorithm"], + "commit_id": identity["source_commit_id"], + "tree_id": identity["source_tree_id"], + } + ): + raise ReplayError("materializer response does not match the input snapshot") return { "response_sha256": digest_bytes(result.stdout), - "receipt_sha256": response["payloads"][0]["sha256"], + "receipt_sha256": receipt_sha, "candidate_commit_id": candidate["commit_id"], "candidate_tree_id": candidate["tree_id"], } @@ -226,12 +252,15 @@ def result(state): def replay(arguments): repository = Path(__file__).resolve().parents[2] materializer = trusted_file(repository / "adapters/local-git-materializer/v1/materialize.sh") - input_value, input_sha = read_json(arguments.input, MAX_INPUT_BYTES) - identity = input_identity(input_value, input_sha, arguments, materializer) state_dir = private_directory(arguments.state_dir) disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) state_path = state_dir / "run.json" + input_snapshot_path = state_dir / "materialization-input.json" lock_path = state_dir / "replay.lock" + input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) + input_value = parse_json(input_bytes) + input_sha = digest_bytes(input_bytes) + identity = input_identity(input_value, input_sha, arguments, materializer) interrupted = {"value": False} previous_term = signal.getsignal(signal.SIGTERM) previous_int = signal.getsignal(signal.SIGINT) @@ -252,7 +281,12 @@ def replay(arguments): if state is None: state = {"schema_version": 1, "kind": "delivery_replay_state", "identity": identity, "phase": "materializing", "authority": "none", "qualification": "unavailable"} + atomic_bytes(input_snapshot_path, input_bytes) atomic_json(state_path, state) + elif not input_snapshot_path.is_file() or input_snapshot_path.is_symlink() or ( + digest_bytes(read_bytes(input_snapshot_path, MAX_INPUT_BYTES)) != identity["input_sha256"] + ): + raise ReplayError("saved materialization input snapshot is unavailable") if state["phase"] == "failed": if state.get("recoverable"): state["recovery"] = "start a new replay with fresh empty candidate, scratch, and state directories" @@ -272,7 +306,7 @@ def replay(arguments): return 0 if state["phase"] == "materializing": try: - state["materialization"] = run_materializer(arguments, materializer) + state["materialization"] = run_materializer(arguments, materializer, input_snapshot_path, identity) except ReplayError as error: state.update({"phase": "failed", "recoverable": True, "reason": str(error)}) atomic_json(state_path, state) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 46de5f8..1a9aba9 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -81,6 +81,35 @@ request_sha=$(jq -r '.identity.request_sha256' "$tmp/changed-state/run.json") candidate_tree=$(jq -r '.identity.candidate_tree_id' "$tmp/changed-state/run.json") pass 'changed materialization and fixed read-only verification wait for review' +cp "$base_input" "$tmp/mutable-input.json" +mkdir -m 700 "$tmp/mutation-state" "$tmp/mutation-candidate" "$tmp/mutation-scratch" +python3 "$replay" --input "$tmp/mutable-input.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/mutation-candidate" --scratch-root "$tmp/mutation-scratch" --state-dir "$tmp/mutation-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/mutation.out" & +mutation_pid=$! +mutation_wait=0 +while [ ! -f "$tmp/mutation-state/materialization-input.json" ]; do + if ! kill -0 "$mutation_pid" 2>/dev/null; then + wait "$mutation_pid" || : + sed -n '1,12p' "$tmp/mutation.out" >&2 + fail input-snapshot-start + fi + mutation_wait=$((mutation_wait + 1)) + if [ "$mutation_wait" -gt 100 ]; then + kill -TERM "$mutation_pid" 2>/dev/null || : + wait "$mutation_pid" || : + fail input-snapshot-timeout + fi + sleep 0.1 +done +printf '%s\n' '{"replaced":"after snapshot"}' >"$tmp/mutable-input.json" +wait "$mutation_pid" || fail input-snapshot-run +[ "$(sha_file "$tmp/mutation-state/materialization-input.json")" = "$(sha_file "$base_input")" ] || fail input-snapshot-bytes +jq -e '.state.phase=="review-wait" and .state.identity.input_sha256==$sha' --arg sha "$(sha_file "$base_input")" \ + "$tmp/mutation.out" >/dev/null || fail input-snapshot-output +pass 'replacement of the original input after snapshot cannot change materialization' + printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ From 0462ad2a9c62d96d8347988326d3469cd75af27b Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 19:14:36 -0400 Subject: [PATCH 05/16] Reconcile interrupted delivery materialization --- delivery/v1/replay.py | 56 ++++++++++++++++++-- scripts/test/delivery-replay.test.sh | 78 ++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 4 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index a241f82..431673f 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -9,6 +9,7 @@ from pathlib import Path import re import signal +import shutil import subprocess import sys import tempfile @@ -55,7 +56,7 @@ def read_bytes(path, limit): def parse_json(data): try: return json.loads(data) - except json.JSONDecodeError as error: + except (json.JSONDecodeError, UnicodeDecodeError) as error: raise ReplayError("input is not JSON") from error @@ -168,11 +169,13 @@ def input_identity(input_value, input_sha, arguments, materializer): return identity -def run_materializer(arguments, materializer, input_path, identity): +def run_materializer(arguments, materializer, input_path, identity, candidate_root=None, scratch_root=None): + candidate_root = Path(arguments.candidate_root).resolve() if candidate_root is None else candidate_root + scratch_root = Path(arguments.scratch_root).resolve() if scratch_root is None else scratch_root command = [ str(materializer), "materialize", str(input_path), arguments.source_repository_id, str(Path(arguments.source_git_dir).resolve()), - str(Path(arguments.candidate_root).resolve()), str(Path(arguments.scratch_root).resolve()), + str(candidate_root), str(scratch_root), str(Path(arguments.closure_helper).resolve()), str(Path(arguments.jq_bin).resolve()), ] environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"} @@ -203,11 +206,52 @@ def run_materializer(arguments, materializer, input_path, identity): "receipt_sha256": receipt_sha, "candidate_commit_id": candidate["commit_id"], "candidate_tree_id": candidate["tree_id"], + "candidate_parent_commit_id": candidate["parent_commit_id"], } except (KeyError, IndexError, TypeError, json.JSONDecodeError) as error: raise ReplayError("materializer response is malformed") from error +def candidate_identity(candidate_root): + repository = Path(candidate_root).resolve() / "repository.git" + if not repository.is_dir() or repository.is_symlink(): + return None + environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} + values = [] + for revision in ("refs/heads/candidate", "refs/heads/candidate^{tree}", "refs/heads/candidate^"): + result = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "rev-parse", revision], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + value = result.stdout.decode().strip() + if result.returncode != 0 or not OID.fullmatch(value): + return None + values.append(value) + return {"candidate_commit_id": values[0], "candidate_tree_id": values[1], + "candidate_parent_commit_id": values[2]} + + +def reconcile_materialization(arguments, materializer, input_path, identity, state_dir): + existing = candidate_identity(arguments.candidate_root) + if existing is None: + return None + recovery_candidate = Path(tempfile.mkdtemp(prefix="reconcile-candidate-", dir=state_dir)) + recovery_scratch = Path(tempfile.mkdtemp(prefix="reconcile-scratch-", dir=state_dir)) + try: + recomputed = run_materializer(arguments, materializer, input_path, identity, + recovery_candidate, recovery_scratch) + finally: + shutil.rmtree(recovery_candidate, ignore_errors=True) + shutil.rmtree(recovery_scratch, ignore_errors=True) + if existing != { + "candidate_commit_id": recomputed["candidate_commit_id"], + "candidate_tree_id": recomputed["candidate_tree_id"], + "candidate_parent_commit_id": recomputed["candidate_parent_commit_id"], + }: + raise ReplayError("existing candidate does not match frozen materialization input") + return recomputed + + def verify_candidate(candidate_root, candidate_tree, path, expected): repository = Path(candidate_root).resolve() / "repository.git" if not repository.is_dir() or repository.is_symlink() or not OID.fullmatch(candidate_tree): @@ -306,7 +350,11 @@ def replay(arguments): return 0 if state["phase"] == "materializing": try: - state["materialization"] = run_materializer(arguments, materializer, input_snapshot_path, identity) + reconciled = reconcile_materialization(arguments, materializer, input_snapshot_path, + identity, state_dir) + state["materialization"] = reconciled or run_materializer( + arguments, materializer, input_snapshot_path, identity + ) except ReplayError as error: state.update({"phase": "failed", "recoverable": True, "reason": str(error)}) atomic_json(state_path, state) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 1a9aba9..0c371b8 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -110,6 +110,84 @@ jq -e '.state.phase=="review-wait" and .state.identity.input_sha256==$sha' --arg "$tmp/mutation.out" >/dev/null || fail input-snapshot-output pass 'replacement of the original input after snapshot cannot change materialization' +kill_wrapper="$tmp/kill-after-materialize.py" +printf '%s\n' \ + 'import importlib.util, os, signal, sys' \ + 'path, arguments = sys.argv[1], sys.argv[2:]' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'original = module.run_materializer' \ + 'def stop_after_materialization(*args, **kwargs):' \ + ' result = original(*args, **kwargs)' \ + ' os.kill(os.getpid(), signal.SIGKILL)' \ + ' return result' \ + 'module.run_materializer = stop_after_materialization' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$kill_wrapper" +mkdir -m 700 "$tmp/reconcile-state" "$tmp/reconcile-candidate" "$tmp/reconcile-scratch" +if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/reconcile-killed.out" 2>&1; then fail reconcile-kill; fi +[ "$(jq -r '.phase' "$tmp/reconcile-state/run.json")" = materializing ] && [ -d "$tmp/reconcile-candidate/repository.git" ] || + fail reconcile-window +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/reconcile-retry.out" +jq -e '.state.phase=="review-wait"' "$tmp/reconcile-retry.out" >/dev/null || fail reconcile-retry +pass 'SIGKILL after materializer output reconciles the existing candidate once' + +mkdir -m 700 "$tmp/reconcile-bad-state" "$tmp/reconcile-bad-candidate" "$tmp/reconcile-bad-scratch" +if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-bad-candidate" --scratch-root "$tmp/reconcile-bad-scratch" --state-dir "$tmp/reconcile-bad-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/reconcile-bad-killed.out" 2>&1; then fail reconcile-bad-kill; fi +bad_repo="$tmp/reconcile-bad-candidate/repository.git" +bad_commit=$(printf '%s\n' mismatch | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ + GIT_COMMITTER_EMAIL=fixture@example.invalid /usr/bin/git --git-dir="$bad_repo" commit-tree "$source_tree" -p "$source_commit") +/usr/bin/git --git-dir="$bad_repo" update-ref refs/heads/candidate "$bad_commit" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-bad-candidate" --scratch-root "$tmp/reconcile-bad-scratch" --state-dir "$tmp/reconcile-bad-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/reconcile-bad.out" 2>&1; then fail reconcile-mismatch; fi +jq -e '.state.phase=="failed" and (.state.reason|contains("does not match frozen"))' "$tmp/reconcile-bad.out" >/dev/null || + fail reconcile-mismatch-state +pass 'a mismatched interrupted candidate is rejected without cleanup' + +printf '\377' >"$tmp/invalid-input.json" +mkdir -m 700 "$tmp/invalid-input-state" "$tmp/invalid-input-candidate" "$tmp/invalid-input-scratch" +if python3 "$replay" --input "$tmp/invalid-input.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/invalid-input-candidate" --scratch-root "$tmp/invalid-input-scratch" --state-dir "$tmp/invalid-input-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/invalid-input.out" 2>&1; then fail invalid-utf8-input; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/invalid-input.out" || + grep -Fq Traceback "$tmp/invalid-input.out"; then + fail invalid-utf8-input-error +fi +printf '\377' >"$tmp/reconcile-state/invalid-review.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/reconcile-state/invalid-review.json" >"$tmp/invalid-review.out" 2>&1; then fail invalid-utf8-review; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/invalid-review.out" || + grep -Fq Traceback "$tmp/invalid-review.out"; then + fail invalid-utf8-review-error +fi +mkdir -m 700 "$tmp/invalid-journal-state" "$tmp/invalid-journal-candidate" "$tmp/invalid-journal-scratch" +printf '\377' >"$tmp/invalid-journal-state/run.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/invalid-journal-candidate" --scratch-root "$tmp/invalid-journal-scratch" --state-dir "$tmp/invalid-journal-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/invalid-journal.out" 2>&1; then fail invalid-utf8-journal; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/invalid-journal.out" || + grep -Fq Traceback "$tmp/invalid-journal.out"; then + fail invalid-utf8-journal-error +fi +pass 'invalid UTF-8 input, review, and journal records fail without a traceback' + printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ From 4c29c1a321aeeff111f7442f1c561da9ab907c3c Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 20:07:26 -0400 Subject: [PATCH 06/16] Validate replay journal recovery states --- delivery/v1/replay.py | 76 +++++++++++++++++++++--- scripts/test/delivery-replay.test.sh | 88 ++++++++++++++++++++++++---- 2 files changed, 147 insertions(+), 17 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 431673f..706eddf 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -212,7 +212,7 @@ def run_materializer(arguments, materializer, input_path, identity, candidate_ro raise ReplayError("materializer response is malformed") from error -def candidate_identity(candidate_root): +def candidate_identity(candidate_root, source_commit): repository = Path(candidate_root).resolve() / "repository.git" if not repository.is_dir() or repository.is_symlink(): return None @@ -220,19 +220,27 @@ def candidate_identity(candidate_root): "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} values = [] - for revision in ("refs/heads/candidate", "refs/heads/candidate^{tree}", "refs/heads/candidate^"): + for revision in ("refs/heads/candidate", "refs/heads/candidate^{tree}"): result = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "rev-parse", revision], env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) value = result.stdout.decode().strip() if result.returncode != 0 or not OID.fullmatch(value): return None values.append(value) + if values[0] == source_commit: + return {"candidate_commit_id": values[0], "candidate_tree_id": values[1], + "candidate_parent_commit_id": source_commit} + result = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "rev-parse", "refs/heads/candidate^"], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + parent = result.stdout.decode().strip() + if result.returncode != 0 or not OID.fullmatch(parent): + return None return {"candidate_commit_id": values[0], "candidate_tree_id": values[1], - "candidate_parent_commit_id": values[2]} + "candidate_parent_commit_id": parent} def reconcile_materialization(arguments, materializer, input_path, identity, state_dir): - existing = candidate_identity(arguments.candidate_root) + existing = candidate_identity(arguments.candidate_root, identity["source_commit_id"]) if existing is None: return None recovery_candidate = Path(tempfile.mkdtemp(prefix="reconcile-candidate-", dir=state_dir)) @@ -287,6 +295,61 @@ def observation(path, kind, identity, field): return {"actor_id": value["actor_id"], field: value.get(field), "sha256": source_sha} +def validate_state(state, identity): + if not isinstance(state, dict) or state.get("schema_version") != 1 or \ + state.get("kind") != "delivery_replay_state" or state.get("authority") != "none" or \ + state.get("qualification") != "unavailable": + raise ReplayError("state journal is malformed") + saved = state.get("identity") + if not isinstance(saved, dict) or any( + not isinstance(saved.get(name), str) or not re.fullmatch(r"[0-9a-f]{64}", saved[name]) + for name in ("input_sha256", "request_sha256", "materializer_sha256", "closure_helper_sha256", "jq_sha256", "run_key") + ) or not isinstance(saved.get("source_repository_id"), str) or \ + saved.get("source_hash_algorithm") not in {"sha1", "sha256"} or \ + any(not isinstance(saved.get(name), str) or not OID.fullmatch(saved[name]) + for name in ("source_commit_id", "source_tree_id")) or \ + not isinstance(saved.get("verifier"), dict) or \ + not isinstance(saved["verifier"].get("id"), str) or \ + not isinstance(saved["verifier"].get("path"), str) or \ + not re.fullmatch(r"[0-9a-f]{64}", str(saved["verifier"].get("expected_sha256", ""))): + raise ReplayError("state journal identity is malformed") + for name in ("candidate_commit_id", "candidate_tree_id"): + if name in saved and (not isinstance(saved[name], str) or not OID.fullmatch(saved[name])): + raise ReplayError("state journal candidate identity is malformed") + phase = state.get("phase") + if phase not in {"materializing", "verifying", "review-wait", "publish-wait", "completed-offline", "failed"}: + raise ReplayError("state journal phase is malformed") + needs_materialization = phase in {"verifying", "review-wait", "publish-wait", "completed-offline"} + materialization = state.get("materialization") + if needs_materialization and (not isinstance(materialization, dict) or any( + not isinstance(materialization.get(name), str) or not OID.fullmatch(materialization[name]) + for name in ("candidate_commit_id", "candidate_tree_id", "candidate_parent_commit_id") + ) or any( + not isinstance(materialization.get(name), str) or not re.fullmatch(r"[0-9a-f]{64}", materialization[name]) + for name in ("response_sha256", "receipt_sha256") + )): + raise ReplayError("state journal materialization is malformed") + if phase in {"review-wait", "publish-wait", "completed-offline"}: + verification = state.get("verification") + if not isinstance(verification, dict) or not isinstance(verification.get("id"), str) or \ + not isinstance(verification.get("path"), str) or \ + not re.fullmatch(r"[0-9a-f]{64}", str(verification.get("sha256", ""))): + raise ReplayError("state journal verification is malformed") + if phase in {"publish-wait", "completed-offline"}: + review = state.get("review") + if not isinstance(review, dict) or not ACTOR.fullmatch(str(review.get("actor_id", ""))) or \ + review.get("verdict") != "clean" or not re.fullmatch(r"[0-9a-f]{64}", str(review.get("sha256", ""))): + raise ReplayError("state journal review is malformed") + if phase == "completed-offline": + publisher = state.get("publisher") + if not isinstance(publisher, dict) or not ACTOR.fullmatch(str(publisher.get("actor_id", ""))) or \ + publisher.get("disposition") != "offline-simulated" or \ + not re.fullmatch(r"[0-9a-f]{64}", str(publisher.get("sha256", ""))): + raise ReplayError("state journal publisher is malformed") + if phase == "failed" and not isinstance(state.get("reason"), str): + raise ReplayError("state journal failure is malformed") + + def result(state): print(json.dumps({"kind": "delivery_replay_receipt", "authority": "none", "qualification": "unavailable", "offline_simulation": True, @@ -317,9 +380,8 @@ def replay(arguments): state = None if state_path.exists(): state, _ = read_json(state_path, MAX_OBSERVATION_BYTES) - if not isinstance(state, dict): - raise ReplayError("state journal is malformed") - if state is not None and state.get("identity", {}).get("run_key") != identity["run_key"]: + validate_state(state, identity) + if state is not None and any(state["identity"].get(name) != value for name, value in identity.items()): result({"phase": "stale", "reason": "run identity changed"}) return 2 if state is None: diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 0c371b8..fda4949 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2016 set -euo pipefail export LC_ALL=C export PYTHONDONTWRITEBYTECODE=1 @@ -59,6 +60,36 @@ make_source() { printf '%s %s\n' "$commit" "$tree" } +make_source_with_ancestor() { + local destination=$1 blob tree base commit + /bin/mkdir -m 700 "$destination" + git_clean init -q --bare "$destination" + blob=$(printf '%s\n' alpha beta | git_clean --git-dir="$destination" hash-object -w --stdin) + tree=$(printf '100644 blob %s\tsource.txt\n' "$blob" | git_clean --git-dir="$destination" mktree) + base=$(printf '%s\n' base | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_AUTHOR_NAME=fixture \ + GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ + GIT_COMMITTER_EMAIL=fixture@example.invalid /usr/bin/git --git-dir="$destination" commit-tree "$tree") + commit=$(printf '%s\n' source | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_AUTHOR_NAME=fixture \ + GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ + GIT_COMMITTER_EMAIL=fixture@example.invalid /usr/bin/git --git-dir="$destination" commit-tree "$tree" -p "$base") + git_clean --git-dir="$destination" update-ref refs/heads/main "$commit" + printf '%s %s\n' "$commit" "$tree" +} + +make_empty_input() { + local input=$1 output=$2 + local intermediate="$output.intermediate" request="$output.request" + "$jq_bin" -S -c '(.stage_request.content.body.inputs[] | select(.input_id=="input.producer-patch") | .value.value.value.sha256) = $sha | + (.payloads[] | select(.input_id=="input.producer-patch") | .data) = "" | + (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .content.data) = "" | + (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .sha256) = $sha' \ + --arg sha "$(printf '' | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}')" "$input" >"$intermediate" + "$jq_bin" -S -c '.stage_request.content' "$intermediate" >"$request" + "$jq_bin" -S -c --arg sha "$(sha_file "$request")" '.stage_request.sha256=$sha' "$intermediate" >"$output" +} + read -r source_commit source_tree < <(make_source "$tmp/source.git") "$fixture_builder" build "$tmp/fixture" "$jq_bin" sha1 "$source_commit" "$source_tree" base_input="$tmp/fixture/input.json" @@ -195,6 +226,29 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits +expect_malformed_state() { + local name=$1 filter=$2 + local state_root="$tmp/malformed-$name-state" + /bin/mkdir -m 700 "$state_root" + cp "$tmp/changed-state/materialization-input.json" "$state_root/materialization-input.json" + jq -S -c "$filter" "$tmp/changed-state/run.json" >"$state_root/run.json" + if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$state_root" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/malformed-$name.out" 2>&1; then fail "malformed-$name"; fi + if ! grep -Fq 'delivery replay: state journal' "$tmp/malformed-$name.out" || + grep -Fq Traceback "$tmp/malformed-$name.out"; then + fail "malformed-$name-error" + fi +} +expect_malformed_state identity-type '.identity=[]' +expect_malformed_state missing-phase 'del(.phase)' +expect_malformed_state invalid-phase '.phase="unknown"' +expect_malformed_state missing-materialization '(.phase="verifying") | del(.materialization)' +expect_malformed_state missing-verification '(.phase="review-wait") | del(.verification)' +expect_malformed_state missing-review '(.phase="publish-wait") | del(.review)' +expect_malformed_state missing-publisher '(.phase="completed-offline") | del(.publisher)' +pass 'malformed state phases and nested records fail without a traceback' jq -S -c '.note="changed after review wait"' "$tmp/review.json" >"$tmp/changed-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ @@ -234,22 +288,36 @@ if python3 "$replay" --input "$base_input" --source-repository-id fixture.target grep -Fq 'does not match this candidate' "$tmp/mismatch.out" || fail mismatched-review-error pass 'mismatched supplied review cannot complete the replay' -empty_input="$tmp/empty-input.json" -jq -S -c '(.stage_request.content.body.inputs[] | select(.input_id=="input.producer-patch") | .value.value.value.sha256) = $sha | - (.payloads[] | select(.input_id=="input.producer-patch") | .data) = "" | - (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .content.data) = "" | - (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .sha256) = $sha' \ - --arg sha "$(printf '' | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}')" "$base_input" >"$empty_input" -empty_request="$tmp/empty-request.json" -jq -S -c '.stage_request.content' "$empty_input" >"$empty_request" -empty_request_sha=$(sha_file "$empty_request") -jq -S -c --arg sha "$empty_request_sha" '.stage_request.sha256=$sha' "$empty_input" >"$tmp/empty-final.json" +make_empty_input "$base_input" "$tmp/empty-final.json" source_digest=$(printf '%s\n' alpha beta | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}') run_replay no-change "$tmp/empty-final.json" "$source_digest" >"$tmp/no-change.out" jq -e '.state.phase=="review-wait" and .state.materialization.candidate_tree_id==.state.identity.source_tree_id' "$tmp/no-change.out" >/dev/null || fail no-change pass 'empty producer patch records a no-change candidate before review' +recover_no_change() { + local name=$1 input=$2 source=$3 + local state="$tmp/$name-state" candidate="$tmp/$name-candidate" scratch="$tmp/$name-scratch" + /bin/mkdir -m 700 "$state" "$candidate" "$scratch" + if python3 "$kill_wrapper" "$replay" --input "$input" --source-repository-id fixture.target --source-git-dir "$source" \ + --candidate-root "$candidate" --scratch-root "$scratch" --state-dir "$state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$source_digest" \ + >"$tmp/$name-killed.out" 2>&1; then fail "$name-kill"; fi + [ "$(jq -r '.phase' "$state/run.json")" = materializing ] && [ -d "$candidate/repository.git" ] || fail "$name-window" + python3 "$replay" --input "$input" --source-repository-id fixture.target --source-git-dir "$source" \ + --candidate-root "$candidate" --scratch-root "$scratch" --state-dir "$state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$source_digest" \ + >"$tmp/$name-retry.out" + jq -e '.state.phase=="review-wait" and .state.materialization.candidate_commit_id==.state.identity.source_commit_id' \ + "$tmp/$name-retry.out" >/dev/null || fail "$name-retry" +} +recover_no_change no-change-root "$tmp/empty-final.json" "$tmp/source.git" +read -r ancestor_commit ancestor_tree < <(make_source_with_ancestor "$tmp/ancestor-source.git") +"$fixture_builder" build "$tmp/ancestor-fixture" "$jq_bin" sha1 "$ancestor_commit" "$ancestor_tree" +make_empty_input "$tmp/ancestor-fixture/input.json" "$tmp/ancestor-empty.json" +recover_no_change no-change-ancestor "$tmp/ancestor-empty.json" "$tmp/ancestor-source.git" +pass 'SIGKILL no-change recovery accepts both root and ancestor source commits' + mkdir -m 700 "$tmp/interrupted-state" "$tmp/interrupted-candidate" "$tmp/interrupted-scratch" python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/interrupted-candidate" --scratch-root "$tmp/interrupted-scratch" --state-dir "$tmp/interrupted-state" \ From e02c6346a59cd4e6c35f97f9cd358a2df24ce40f Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 20:57:46 -0400 Subject: [PATCH 07/16] fix replay identity and journal validation --- delivery/v1/replay.py | 95 ++++++++++++++++++++++------ scripts/test/delivery-replay.test.sh | 63 ++++++++++++++++++ 2 files changed, 140 insertions(+), 18 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 706eddf..05791fe 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -20,6 +20,21 @@ MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 OID = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}\Z") ACTOR = re.compile(r"[a-z0-9][a-z0-9._:-]{0,127}\Z") +PACKAGE_FILES = ( + "adapters/local-git-materializer/v1/materialize.sh", + "adapters/local-git-materializer/v1/protocol.jq", + "scripts/core-contract.sh", + "core/v2/generation-registry.json", +) +GENERATION_FILES = ( + "core-ingress.sh", + "contracts.jq", + "modules/schema.jq", + "modules/profile_graph.jq", + "modules/stage_request.jq", + "modules/result_facts.jq", + "modules/result_truth.jq", +) class ReplayError(Exception): @@ -60,11 +75,6 @@ def parse_json(data): raise ReplayError("input is not JSON") from error -def read_json(path, limit): - data = read_bytes(path, limit) - return parse_json(data), digest_bytes(data) - - def private_directory(path): value = Path(path) stat = value.stat() @@ -124,7 +134,30 @@ def safe_path(value): return value -def input_identity(input_value, input_sha, arguments, materializer): +def package_paths(generation): + root = f"core/v2/generations/{generation}" + return PACKAGE_FILES + tuple(f"{root}/{name}" for name in GENERATION_FILES) + + +def materializer_package_identity(repository): + core_path = trusted_file(repository / "scripts/core-contract.sh") + core_bytes = core_path.read_bytes() + match = re.search( + rb"^PORTABLE_CORE_GENERATION='(g-[0-9a-f]{64})'$", core_bytes, re.MULTILINE + ) + if match is None: + raise ReplayError("materializer package generation is unavailable") + generation = match.group(1).decode() + files = { + relative: digest_bytes(trusted_file(repository / relative).read_bytes()) + for relative in package_paths(generation) + } + package = {"generation_id": generation, "files": files} + package["sha256"] = digest_bytes(canonical(package)) + return package + + +def input_identity(input_value, input_sha, arguments, repository): try: request = input_value["stage_request"] request_sha = request["sha256"] @@ -148,7 +181,7 @@ def input_identity(input_value, input_sha, arguments, materializer): raise ReplayError("expected verifier digest is invalid") closure = trusted_file(arguments.closure_helper) jq_bin = trusted_file(arguments.jq_bin) - materializer_sha = digest_bytes(materializer.read_bytes()) + package = materializer_package_identity(repository) identity = { "input_sha256": input_sha, "request_sha256": request_sha, @@ -160,7 +193,9 @@ def input_identity(input_value, input_sha, arguments, materializer): "path": safe_path(arguments.verify_path), "expected_sha256": expected, }, - "materializer_sha256": materializer_sha, + "driver_sha256": digest_bytes(trusted_file(Path(__file__).resolve()).read_bytes()), + "materializer_sha256": package["files"][PACKAGE_FILES[0]], + "materializer_package": package, "closure_helper_sha256": digest_bytes(closure.read_bytes()), "jq_sha256": digest_bytes(jq_bin.read_bytes()), "source_repository_id": arguments.source_repository_id, @@ -285,10 +320,12 @@ def verify_candidate(candidate_root, candidate_tree, path, expected): def observation(path, kind, identity, field): if path is None: return None - value, source_sha = read_json(path, MAX_OBSERVATION_BYTES) + source = read_bytes(path, MAX_OBSERVATION_BYTES) + value = parse_json(source) + source_sha = digest_bytes(source) if not isinstance(value, dict) or value.get("schema_version") != 1 or value.get("kind") != kind: raise ReplayError("offline observation is malformed") - if not ACTOR.fullmatch(str(value.get("actor_id", ""))): + if not isinstance(value.get("actor_id"), str) or not ACTOR.fullmatch(value["actor_id"]): raise ReplayError("offline observation actor is invalid") if value.get("request_sha256") != identity["request_sha256"] or value.get("candidate_tree_id") != identity["candidate_tree_id"]: raise ReplayError("offline observation does not match this candidate") @@ -303,7 +340,8 @@ def validate_state(state, identity): saved = state.get("identity") if not isinstance(saved, dict) or any( not isinstance(saved.get(name), str) or not re.fullmatch(r"[0-9a-f]{64}", saved[name]) - for name in ("input_sha256", "request_sha256", "materializer_sha256", "closure_helper_sha256", "jq_sha256", "run_key") + for name in ("input_sha256", "request_sha256", "driver_sha256", "materializer_sha256", + "closure_helper_sha256", "jq_sha256", "run_key") ) or not isinstance(saved.get("source_repository_id"), str) or \ saved.get("source_hash_algorithm") not in {"sha1", "sha256"} or \ any(not isinstance(saved.get(name), str) or not OID.fullmatch(saved[name]) @@ -313,13 +351,27 @@ def validate_state(state, identity): not isinstance(saved["verifier"].get("path"), str) or \ not re.fullmatch(r"[0-9a-f]{64}", str(saved["verifier"].get("expected_sha256", ""))): raise ReplayError("state journal identity is malformed") - for name in ("candidate_commit_id", "candidate_tree_id"): - if name in saved and (not isinstance(saved[name], str) or not OID.fullmatch(saved[name])): - raise ReplayError("state journal candidate identity is malformed") + package = saved.get("materializer_package") + if not isinstance(package, dict) or not isinstance(package.get("generation_id"), str) or \ + not re.fullmatch(r"g-[0-9a-f]{64}", package["generation_id"]) or \ + not isinstance(package.get("files"), dict) or \ + set(package["files"]) != set(package_paths(package["generation_id"])) or any( + not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) + for value in package["files"].values() + ) or not isinstance(package.get("sha256"), str) or \ + package["sha256"] != digest_bytes(canonical({ + "generation_id": package["generation_id"], "files": package["files"] + })) or saved["materializer_sha256"] != package["files"][PACKAGE_FILES[0]]: + raise ReplayError("state journal materializer package is malformed") phase = state.get("phase") if phase not in {"materializing", "verifying", "review-wait", "publish-wait", "completed-offline", "failed"}: raise ReplayError("state journal phase is malformed") needs_materialization = phase in {"verifying", "review-wait", "publish-wait", "completed-offline"} + for name in ("candidate_commit_id", "candidate_tree_id"): + if (needs_materialization and name not in saved) or ( + name in saved and (not isinstance(saved[name], str) or not OID.fullmatch(saved[name])) + ): + raise ReplayError("state journal candidate identity is malformed") materialization = state.get("materialization") if needs_materialization and (not isinstance(materialization, dict) or any( not isinstance(materialization.get(name), str) or not OID.fullmatch(materialization[name]) @@ -329,6 +381,11 @@ def validate_state(state, identity): for name in ("response_sha256", "receipt_sha256") )): raise ReplayError("state journal materialization is malformed") + if needs_materialization and any( + saved[name] != materialization[name] + for name in ("candidate_commit_id", "candidate_tree_id") + ): + raise ReplayError("state journal candidate identity does not match materialization") if phase in {"review-wait", "publish-wait", "completed-offline"}: verification = state.get("verification") if not isinstance(verification, dict) or not isinstance(verification.get("id"), str) or \ @@ -337,12 +394,14 @@ def validate_state(state, identity): raise ReplayError("state journal verification is malformed") if phase in {"publish-wait", "completed-offline"}: review = state.get("review") - if not isinstance(review, dict) or not ACTOR.fullmatch(str(review.get("actor_id", ""))) or \ + if not isinstance(review, dict) or not isinstance(review.get("actor_id"), str) or \ + not ACTOR.fullmatch(review["actor_id"]) or \ review.get("verdict") != "clean" or not re.fullmatch(r"[0-9a-f]{64}", str(review.get("sha256", ""))): raise ReplayError("state journal review is malformed") if phase == "completed-offline": publisher = state.get("publisher") - if not isinstance(publisher, dict) or not ACTOR.fullmatch(str(publisher.get("actor_id", ""))) or \ + if not isinstance(publisher, dict) or not isinstance(publisher.get("actor_id"), str) or \ + not ACTOR.fullmatch(publisher["actor_id"]) or \ publisher.get("disposition") != "offline-simulated" or \ not re.fullmatch(r"[0-9a-f]{64}", str(publisher.get("sha256", ""))): raise ReplayError("state journal publisher is malformed") @@ -367,7 +426,7 @@ def replay(arguments): input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) input_value = parse_json(input_bytes) input_sha = digest_bytes(input_bytes) - identity = input_identity(input_value, input_sha, arguments, materializer) + identity = input_identity(input_value, input_sha, arguments, repository) interrupted = {"value": False} previous_term = signal.getsignal(signal.SIGTERM) previous_int = signal.getsignal(signal.SIGINT) @@ -379,7 +438,7 @@ def replay(arguments): fcntl.flock(lock, fcntl.LOCK_EX) state = None if state_path.exists(): - state, _ = read_json(state_path, MAX_OBSERVATION_BYTES) + state = parse_json(read_bytes(state_path, MAX_OBSERVATION_BYTES)) validate_state(state, identity) if state is not None and any(state["identity"].get(name) != value for name, value in identity.items()): result({"phase": "stale", "reason": "run identity changed"}) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index fda4949..1838785 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -37,6 +37,9 @@ else fi /bin/chmod 0555 "$runtime/jq" jq_bin="$runtime/jq" +PATH="$runtime:/usr/bin:/bin" +export PATH +[ "$(command -v jq)" = "$runtime/jq" ] || fail 'private jq is not first on PATH' /usr/bin/cc -std=c11 -Wall -Wextra -Werror -O2 "$closure_source" -o "$runtime/object-closure" /bin/chmod 0555 "$runtime/object-closure" @@ -221,11 +224,24 @@ pass 'invalid UTF-8 input, review, and journal records fail without a traceback' printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/numeric-review.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/numeric-review.json" >"$tmp/numeric-review.out" 2>&1; then fail numeric-review-actor; fi +grep -Fq 'offline observation actor is invalid' "$tmp/numeric-review.out" || fail numeric-review-actor-error +pass 'offline observations require a string actor identity' python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/numeric-publisher.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --publisher-observation "$tmp/numeric-publisher.json" >"$tmp/numeric-publisher.out" 2>&1; then fail numeric-publisher-actor; fi +grep -Fq 'offline observation actor is invalid' "$tmp/numeric-publisher.out" || fail numeric-publisher-actor-error expect_malformed_state() { local name=$1 filter=$2 local state_root="$tmp/malformed-$name-state" @@ -248,6 +264,19 @@ expect_malformed_state missing-materialization '(.phase="verifying") | del(.mate expect_malformed_state missing-verification '(.phase="review-wait") | del(.verification)' expect_malformed_state missing-review '(.phase="publish-wait") | del(.review)' expect_malformed_state missing-publisher '(.phase="completed-offline") | del(.publisher)' +for candidate_phase in verifying review-wait publish-wait completed-offline; do + expect_malformed_state "missing-candidate-commit-$candidate_phase" \ + "(.phase=\"$candidate_phase\") | del(.identity.candidate_commit_id)" + expect_malformed_state "missing-candidate-tree-$candidate_phase" \ + "(.phase=\"$candidate_phase\") | del(.identity.candidate_tree_id)" +done +expect_malformed_state mismatched-candidate-commit \ + '(.phase="verifying") | .identity.candidate_commit_id="0000000000000000000000000000000000000000"' +expect_malformed_state mismatched-candidate-tree \ + '(.phase="completed-offline") | .identity.candidate_tree_id="0000000000000000000000000000000000000000"' +expect_malformed_state numeric-review-actor '.review.actor_id=123' +expect_malformed_state numeric-publisher-actor \ + '(.phase="completed-offline") | .publisher={"actor_id":123,"disposition":"offline-simulated","sha256":"0000000000000000000000000000000000000000000000000000000000000000"}' pass 'malformed state phases and nested records fail without a traceback' jq -S -c '.note="changed after review wait"' "$tmp/review.json" >"$tmp/changed-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ @@ -368,4 +397,38 @@ if python3 "$replay" --input "$changed_input" --source-repository-id fixture.tar jq -e '.state.phase=="stale"' "$tmp/changed-input-stale.out" >/dev/null || fail changed-materialization-input-stale-state pass 'changed materialization input cannot reuse the prior run' +package_root="$tmp/replay-package" +generation=$(/usr/bin/sed -n \ + "s/^PORTABLE_CORE_GENERATION='\(g-[0-9a-f]\{64\}\)'$/\1/p" "$root/scripts/core-contract.sh") +/bin/mkdir -p "$package_root/delivery/v1" "$package_root/adapters/local-git-materializer/v1" \ + "$package_root/scripts" "$package_root/core/v2/generations" +/bin/cp "$replay" "$package_root/delivery/v1/replay.py" +/bin/cp "$root/adapters/local-git-materializer/v1/materialize.sh" \ + "$root/adapters/local-git-materializer/v1/protocol.jq" \ + "$package_root/adapters/local-git-materializer/v1/" +/bin/cp "$root/scripts/core-contract.sh" "$package_root/scripts/core-contract.sh" +/bin/cp "$root/core/v2/generation-registry.json" "$package_root/core/v2/generation-registry.json" +/bin/cp -R "$root/core/v2/generations/$generation" "$package_root/core/v2/generations/" +package_replay="$package_root/delivery/v1/replay.py" +/bin/mkdir -m 700 "$tmp/package-state" "$tmp/package-candidate" "$tmp/package-scratch" +python3 "$package_replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/package-candidate" --scratch-root "$tmp/package-scratch" --state-dir "$tmp/package-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/package-first.out" +printf '\n' >>"$package_root/adapters/local-git-materializer/v1/protocol.jq" +if python3 "$package_replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/package-candidate" --scratch-root "$tmp/package-scratch" --state-dir "$tmp/package-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/package-dependency-stale.out"; then fail changed-package-dependency; fi +jq -e '.state.phase=="stale"' "$tmp/package-dependency-stale.out" >/dev/null || fail changed-package-dependency-state +/bin/cp "$root/adapters/local-git-materializer/v1/protocol.jq" \ + "$package_root/adapters/local-git-materializer/v1/protocol.jq" +printf '\n' >>"$package_replay" +if python3 "$package_replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/package-candidate" --scratch-root "$tmp/package-scratch" --state-dir "$tmp/package-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/package-driver-stale.out"; then fail changed-replay-driver; fi +jq -e '.state.phase=="stale"' "$tmp/package-driver-stale.out" >/dev/null || fail changed-replay-driver-state +pass 'changed executable package or replay driver cannot reuse a prior run' + printf 'delivery replay: %s focused checks passed\n' "$passed" From a7de157f1dc7bb0a31b37306a86ddfe33811f97f Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 06:52:53 -0400 Subject: [PATCH 08/16] Fix replay execution identity validation --- delivery/v1/replay.py | 111 ++++++++++++++++---- scripts/test/delivery-replay.test.sh | 145 ++++++++++++++++++++++++++- 2 files changed, 229 insertions(+), 27 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 05791fe..0b11b63 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -71,7 +71,7 @@ def read_bytes(path, limit): def parse_json(data): try: return json.loads(data) - except (json.JSONDecodeError, UnicodeDecodeError) as error: + except (ValueError, UnicodeDecodeError) as error: raise ReplayError("input is not JSON") from error @@ -139,6 +139,40 @@ def package_paths(generation): return PACKAGE_FILES + tuple(f"{root}/{name}" for name in GENERATION_FILES) +def snapshot_file(source, destination, mode): + data = read_bytes(trusted_file(source), MAX_INPUT_BYTES) + destination.parent.mkdir(parents=True, exist_ok=True) + atomic_bytes(destination, data) + os.chmod(destination, mode) + return data + + +def create_execution_snapshot(repository, arguments, state_dir): + root = Path(tempfile.mkdtemp(prefix="execution-", dir=state_dir)) + try: + driver = root / "delivery/v1/replay.py" + snapshot_file(Path(__file__).resolve(), driver, 0o500) + core_relative = "scripts/core-contract.sh" + core = snapshot_file(repository / core_relative, root / core_relative, 0o500) + match = re.search( + rb"^PORTABLE_CORE_GENERATION='(g-[0-9a-f]{64})'$", core, re.MULTILINE + ) + if match is None: + raise ReplayError("materializer package generation is unavailable") + generation = match.group(1).decode() + for relative in package_paths(generation): + if relative == core_relative: + continue + mode = 0o500 if relative.endswith(".sh") else 0o400 + snapshot_file(repository / relative, root / relative, mode) + snapshot_file(arguments.closure_helper, root / ".dependencies/object-closure", 0o500) + snapshot_file(arguments.jq_bin, root / ".dependencies/jq", 0o500) + return root + except (OSError, ReplayError): + shutil.rmtree(root, ignore_errors=True) + raise + + def materializer_package_identity(repository): core_path = trusted_file(repository / "scripts/core-contract.sh") core_bytes = core_path.read_bytes() @@ -157,7 +191,7 @@ def materializer_package_identity(repository): return package -def input_identity(input_value, input_sha, arguments, repository): +def input_identity(input_value, input_sha, arguments, execution): try: request = input_value["stage_request"] request_sha = request["sha256"] @@ -179,9 +213,7 @@ def input_identity(input_value, input_sha, arguments, repository): expected = arguments.expected_sha256 if not re.fullmatch(r"[0-9a-f]{64}", expected): raise ReplayError("expected verifier digest is invalid") - closure = trusted_file(arguments.closure_helper) - jq_bin = trusted_file(arguments.jq_bin) - package = materializer_package_identity(repository) + package = materializer_package_identity(execution) identity = { "input_sha256": input_sha, "request_sha256": request_sha, @@ -193,25 +225,29 @@ def input_identity(input_value, input_sha, arguments, repository): "path": safe_path(arguments.verify_path), "expected_sha256": expected, }, - "driver_sha256": digest_bytes(trusted_file(Path(__file__).resolve()).read_bytes()), + "driver_sha256": digest_bytes(read_bytes(trusted_file(Path(__file__).resolve()), MAX_INPUT_BYTES)), "materializer_sha256": package["files"][PACKAGE_FILES[0]], "materializer_package": package, - "closure_helper_sha256": digest_bytes(closure.read_bytes()), - "jq_sha256": digest_bytes(jq_bin.read_bytes()), + "closure_helper_sha256": digest_bytes(read_bytes( + trusted_file(execution / ".dependencies/object-closure"), MAX_INPUT_BYTES + )), + "jq_sha256": digest_bytes(read_bytes( + trusted_file(execution / ".dependencies/jq"), MAX_INPUT_BYTES + )), "source_repository_id": arguments.source_repository_id, } identity["run_key"] = digest_bytes(canonical(identity)) return identity -def run_materializer(arguments, materializer, input_path, identity, candidate_root=None, scratch_root=None): +def run_materializer(arguments, execution, input_path, identity, candidate_root=None, scratch_root=None): candidate_root = Path(arguments.candidate_root).resolve() if candidate_root is None else candidate_root scratch_root = Path(arguments.scratch_root).resolve() if scratch_root is None else scratch_root command = [ - str(materializer), "materialize", str(input_path), + str(execution / PACKAGE_FILES[0]), "materialize", str(input_path), arguments.source_repository_id, str(Path(arguments.source_git_dir).resolve()), str(candidate_root), str(scratch_root), - str(Path(arguments.closure_helper).resolve()), str(Path(arguments.jq_bin).resolve()), + str(execution / ".dependencies/object-closure"), str(execution / ".dependencies/jq"), ] environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"} result = subprocess.run(command, env=environment, stdout=subprocess.PIPE, @@ -274,14 +310,14 @@ def candidate_identity(candidate_root, source_commit): "candidate_parent_commit_id": parent} -def reconcile_materialization(arguments, materializer, input_path, identity, state_dir): +def reconcile_materialization(arguments, execution, input_path, identity, state_dir): existing = candidate_identity(arguments.candidate_root, identity["source_commit_id"]) if existing is None: return None recovery_candidate = Path(tempfile.mkdtemp(prefix="reconcile-candidate-", dir=state_dir)) recovery_scratch = Path(tempfile.mkdtemp(prefix="reconcile-scratch-", dir=state_dir)) try: - recomputed = run_materializer(arguments, materializer, input_path, identity, + recomputed = run_materializer(arguments, execution, input_path, identity, recovery_candidate, recovery_scratch) finally: shutil.rmtree(recovery_candidate, ignore_errors=True) @@ -388,9 +424,11 @@ def validate_state(state, identity): raise ReplayError("state journal candidate identity does not match materialization") if phase in {"review-wait", "publish-wait", "completed-offline"}: verification = state.get("verification") - if not isinstance(verification, dict) or not isinstance(verification.get("id"), str) or \ - not isinstance(verification.get("path"), str) or \ - not re.fullmatch(r"[0-9a-f]{64}", str(verification.get("sha256", ""))): + if verification != { + "id": saved["verifier"]["id"], + "path": saved["verifier"]["path"], + "sha256": saved["verifier"]["expected_sha256"], + }: raise ReplayError("state journal verification is malformed") if phase in {"publish-wait", "completed-offline"}: review = state.get("review") @@ -416,8 +454,9 @@ def result(state): def replay(arguments): - repository = Path(__file__).resolve().parents[2] - materializer = trusted_file(repository / "adapters/local-git-materializer/v1/materialize.sh") + execution = Path(arguments.execution_root).resolve() + if Path(__file__).resolve() != execution / "delivery/v1/replay.py": + raise ReplayError("execution snapshot is invalid") state_dir = private_directory(arguments.state_dir) disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) state_path = state_dir / "run.json" @@ -426,7 +465,7 @@ def replay(arguments): input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) input_value = parse_json(input_bytes) input_sha = digest_bytes(input_bytes) - identity = input_identity(input_value, input_sha, arguments, repository) + identity = input_identity(input_value, input_sha, arguments, execution) interrupted = {"value": False} previous_term = signal.getsignal(signal.SIGTERM) previous_int = signal.getsignal(signal.SIGINT) @@ -471,10 +510,10 @@ def replay(arguments): return 0 if state["phase"] == "materializing": try: - reconciled = reconcile_materialization(arguments, materializer, input_snapshot_path, + reconciled = reconcile_materialization(arguments, execution, input_snapshot_path, identity, state_dir) state["materialization"] = reconciled or run_materializer( - arguments, materializer, input_snapshot_path, identity + arguments, execution, input_snapshot_path, identity ) except ReplayError as error: state.update({"phase": "failed", "recoverable": True, "reason": str(error)}) @@ -563,8 +602,36 @@ def main(): parser.add_argument("--expected-sha256", required=True) parser.add_argument("--review-observation") parser.add_argument("--publisher-observation") + parser.add_argument("--execution-root", help=argparse.SUPPRESS) + arguments = parser.parse_args() try: - return replay(parser.parse_args()) + if arguments.execution_root is None: + state_dir = private_directory(arguments.state_dir) + execution = create_execution_snapshot(Path(__file__).resolve().parents[2], arguments, state_dir) + child = None + pending_signal = {"value": None} + + def forward_signal(number, _frame): + if child is None: + pending_signal["value"] = number + else: + child.send_signal(number) + + previous_term = signal.signal(signal.SIGTERM, forward_signal) + previous_int = signal.signal(signal.SIGINT, forward_signal) + try: + child = subprocess.Popen([ + sys.executable, str(execution / "delivery/v1/replay.py"), + *sys.argv[1:], "--execution-root", str(execution), + ]) + if pending_signal["value"] is not None: + child.send_signal(pending_signal["value"]) + return child.wait() + finally: + signal.signal(signal.SIGTERM, previous_term) + signal.signal(signal.SIGINT, previous_int) + shutil.rmtree(execution, ignore_errors=True) + return replay(arguments) except (OSError, ReplayError) as error: print(f"delivery replay: {error}", file=sys.stderr) return 1 diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 1838785..1b3f81f 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -9,6 +9,7 @@ root=$(CDPATH='' cd -P -- "${BASH_SOURCE[0]%/*}/../.." && pwd -P) replay="$root/delivery/v1/replay.py" fixture_builder="$root/scripts/test/local-git-materializer-fixtures.sh" closure_source="$root/adapters/local-git-materializer/v1/object-closure.c" +python_with_int_limit=$(command -v python3) tmp=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-delivery-replay.XXXXXX") cleanup() { /bin/rm -rf -- "$tmp"; } trap cleanup EXIT @@ -146,19 +147,26 @@ pass 'replacement of the original input after snapshot cannot change materializa kill_wrapper="$tmp/kill-after-materialize.py" printf '%s\n' \ - 'import importlib.util, os, signal, sys' \ + 'import importlib.util, os, pathlib, signal, sys, types' \ 'path, arguments = sys.argv[1], sys.argv[2:]' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ 'spec.loader.exec_module(module)' \ - 'original = module.run_materializer' \ + 'argument = lambda name: arguments[arguments.index(name) + 1]' \ + 'values = types.SimpleNamespace(closure_helper=argument("--closure-helper"), jq_bin=argument("--jq-bin"))' \ + 'snapshot = module.create_execution_snapshot(pathlib.Path(path).resolve().parents[2], values, pathlib.Path(argument("--state-dir")))' \ + 'snapshot_path = snapshot / "delivery/v1/replay.py"' \ + 'snapshot_spec = importlib.util.spec_from_file_location("snapshot_replay", snapshot_path)' \ + 'snapshot_module = importlib.util.module_from_spec(snapshot_spec)' \ + 'snapshot_spec.loader.exec_module(snapshot_module)' \ + 'original = snapshot_module.run_materializer' \ 'def stop_after_materialization(*args, **kwargs):' \ ' result = original(*args, **kwargs)' \ ' os.kill(os.getpid(), signal.SIGKILL)' \ ' return result' \ - 'module.run_materializer = stop_after_materialization' \ - 'sys.argv = [path] + arguments' \ - 'raise SystemExit(module.main())' >"$kill_wrapper" + 'snapshot_module.run_materializer = stop_after_materialization' \ + 'sys.argv = [str(snapshot_path), *arguments, "--execution-root", str(snapshot)]' \ + 'raise SystemExit(snapshot_module.main())' >"$kill_wrapper" mkdir -m 700 "$tmp/reconcile-state" "$tmp/reconcile-candidate" "$tmp/reconcile-scratch" if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ @@ -191,6 +199,76 @@ jq -e '.state.phase=="failed" and (.state.reason|contains("does not match frozen fail reconcile-mismatch-state pass 'a mismatched interrupted candidate is rejected without cleanup' +for tree_case in numeric list null; do + case "$tree_case" in + numeric) tree_value=123 ;; + list) tree_value='[]' ;; + null) tree_value=null ;; + esac + "$jq_bin" -S -c "(.stage_request.content.body.operation.arguments.source_tree_input_id) as \$id | + (.stage_request.content.body.inputs[] | select(.input_id == \$id) | + .value.value.value.object_id) = $tree_value" "$base_input" >"$tmp/$tree_case-tree.json" + mkdir -m 700 "$tmp/$tree_case-tree-state" "$tmp/$tree_case-tree-candidate" "$tmp/$tree_case-tree-scratch" + if python3 "$replay" --input "$tmp/$tree_case-tree.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/$tree_case-tree-candidate" --scratch-root "$tmp/$tree_case-tree-scratch" \ + --state-dir "$tmp/$tree_case-tree-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/$tree_case-tree.out" 2>&1; then + fail "$tree_case-source-tree" + fi + if ! grep -Fq 'delivery replay: materialization input tree identity is invalid' "$tmp/$tree_case-tree.out" || + grep -Fq Traceback "$tmp/$tree_case-tree.out"; then + fail "$tree_case-source-tree-error" + fi +done +pass 'non-string source tree identities fail without a traceback' + +mkdir -m 700 "$tmp/caller-execution-root" "$tmp/caller-execution-state" \ + "$tmp/caller-execution-candidate" "$tmp/caller-execution-scratch" +printf '%s\n' keep >"$tmp/caller-execution-root/sentinel" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/caller-execution-candidate" --scratch-root "$tmp/caller-execution-scratch" \ + --state-dir "$tmp/caller-execution-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" \ + --execution-root "$tmp/caller-execution-root" >"$tmp/caller-execution.out" 2>&1; then + fail caller-execution-root +fi +[ "$(cat "$tmp/caller-execution-root/sentinel")" = keep ] || fail caller-execution-root-deleted +grep -Fq 'delivery replay: execution snapshot is invalid' "$tmp/caller-execution.out" || + fail caller-execution-root-error +pass 'a caller-supplied execution root is rejected without deleting it' + +huge_integer=$(printf '1%.0s' {1..5000}) +printf '{"huge":%s}\n' "$huge_integer" >"$tmp/huge-input.json" +mkdir -m 700 "$tmp/huge-input-state" "$tmp/huge-input-candidate" "$tmp/huge-input-scratch" +if PYTHONINTMAXSTRDIGITS=4300 "$python_with_int_limit" "$replay" --input "$tmp/huge-input.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/huge-input-candidate" --scratch-root "$tmp/huge-input-scratch" --state-dir "$tmp/huge-input-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/huge-input.out" 2>&1; then fail huge-integer-input; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/huge-input.out" || + grep -Fq Traceback "$tmp/huge-input.out"; then + fail huge-integer-input-error +fi +printf '{"huge":%s}\n' "$huge_integer" >"$tmp/huge-observation.json" +if PYTHONINTMAXSTRDIGITS=4300 "$python_with_int_limit" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/huge-observation.json" >"$tmp/huge-observation.out" 2>&1; then fail huge-integer-observation; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/huge-observation.out" || + grep -Fq Traceback "$tmp/huge-observation.out"; then + fail huge-integer-observation-error +fi +mkdir -m 700 "$tmp/huge-journal-state" "$tmp/huge-journal-candidate" "$tmp/huge-journal-scratch" +printf '{"huge":%s}\n' "$huge_integer" >"$tmp/huge-journal-state/run.json" +if PYTHONINTMAXSTRDIGITS=4300 "$python_with_int_limit" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/huge-journal-candidate" --scratch-root "$tmp/huge-journal-scratch" --state-dir "$tmp/huge-journal-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/huge-journal.out" 2>&1; then fail huge-integer-journal; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/huge-journal.out" || + grep -Fq Traceback "$tmp/huge-journal.out"; then + fail huge-integer-journal-error +fi +pass 'huge JSON integers in input, observation, and journal fail without a traceback' + printf '\377' >"$tmp/invalid-input.json" mkdir -m 700 "$tmp/invalid-input-state" "$tmp/invalid-input-candidate" "$tmp/invalid-input-scratch" if python3 "$replay" --input "$tmp/invalid-input.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ @@ -277,6 +355,14 @@ expect_malformed_state mismatched-candidate-tree \ expect_malformed_state numeric-review-actor '.review.actor_id=123' expect_malformed_state numeric-publisher-actor \ '(.phase="completed-offline") | .publisher={"actor_id":123,"disposition":"offline-simulated","sha256":"0000000000000000000000000000000000000000000000000000000000000000"}' +for verification_phase in review-wait publish-wait completed-offline; do + expect_malformed_state "verification-id-$verification_phase" \ + "(.phase=\"$verification_phase\") | .verification.id=\"delivery.other.v1\"" + expect_malformed_state "verification-path-$verification_phase" \ + "(.phase=\"$verification_phase\") | .verification.path=\"other.txt\"" + expect_malformed_state "verification-sha-$verification_phase" \ + "(.phase=\"$verification_phase\") | .verification.sha256=\"0000000000000000000000000000000000000000000000000000000000000000\"" +done pass 'malformed state phases and nested records fail without a traceback' jq -S -c '.note="changed after review wait"' "$tmp/review.json" >"$tmp/changed-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ @@ -431,4 +517,53 @@ if python3 "$package_replay" --input "$base_input" --source-repository-id fixtur jq -e '.state.phase=="stale"' "$tmp/package-driver-stale.out" >/dev/null || fail changed-replay-driver-state pass 'changed executable package or replay driver cannot reuse a prior run' +/bin/cp "$replay" "$package_replay" +/bin/cp "$root/adapters/local-git-materializer/v1/protocol.jq" \ + "$package_root/adapters/local-git-materializer/v1/protocol.jq" +/bin/cp "$runtime/object-closure" "$tmp/race-object-closure" +/bin/cp "$jq_bin" "$tmp/race-jq" +/bin/chmod 0555 "$tmp/race-object-closure" "$tmp/race-jq" +race_wrapper="$tmp/snapshot-race.py" +printf '%s\n' \ + 'import argparse, importlib.util, os, pathlib, stat, subprocess, sys' \ + 'driver, repository, state_dir, helper, jq_bin, *arguments = sys.argv[1:]' \ + 'spec = importlib.util.spec_from_file_location("replay", driver)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'values = argparse.Namespace(closure_helper=helper, jq_bin=jq_bin)' \ + 'snapshot = module.create_execution_snapshot(pathlib.Path(repository), values, pathlib.Path(state_dir))' \ + 'targets = [pathlib.Path(driver), pathlib.Path(repository) / "adapters/local-git-materializer/v1/protocol.jq", pathlib.Path(helper), pathlib.Path(jq_bin)]' \ + 'saved = [target.read_bytes() for target in targets]' \ + 'modes = [stat.S_IMODE(target.stat().st_mode) for target in targets]' \ + 'try:' \ + ' for target in targets:' \ + ' target.chmod(0o700)' \ + ' target.write_bytes(b"replaced after execution snapshot\n")' \ + ' result = subprocess.run([sys.executable, str(snapshot / "delivery/v1/replay.py"), *arguments, "--execution-root", str(snapshot)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)' \ + 'finally:' \ + ' for target, data, mode in zip(targets, saved, modes):' \ + ' target.write_bytes(data)' \ + ' target.chmod(mode)' \ + 'sys.stdout.buffer.write(result.stdout)' \ + 'sys.stderr.buffer.write(result.stderr)' \ + 'raise SystemExit(result.returncode)' >"$race_wrapper" +/bin/mkdir -m 700 "$tmp/race-state" "$tmp/race-candidate" "$tmp/race-scratch" +driver_sha=$(sha_file "$package_replay") +package_sha=$(sha_file "$package_root/adapters/local-git-materializer/v1/protocol.jq") +helper_sha=$(sha_file "$tmp/race-object-closure") +jq_sha=$(sha_file "$tmp/race-jq") +python3 "$race_wrapper" "$package_replay" "$package_root" "$tmp/race-state" \ + "$tmp/race-object-closure" "$tmp/race-jq" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/race-candidate" --scratch-root "$tmp/race-scratch" --state-dir "$tmp/race-state" \ + --closure-helper "$tmp/race-object-closure" --jq-bin "$tmp/race-jq" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/race.out" +jq -e '.state.phase=="review-wait" and + .state.identity.driver_sha256==$driver and + .state.identity.materializer_package.files["adapters/local-git-materializer/v1/protocol.jq"]==$package and + .state.identity.closure_helper_sha256==$helper and .state.identity.jq_sha256==$jq' \ + --arg driver "$driver_sha" --arg package "$package_sha" --arg helper "$helper_sha" --arg jq "$jq_sha" \ + "$tmp/race.out" >/dev/null || fail immutable-execution-snapshot +pass 'replacement after the private execution snapshot cannot change executed or recorded bytes' + printf 'delivery replay: %s focused checks passed\n' "$passed" From 6d53d29a7b78b6647a570faaa4941b2e5fee8923 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 08:13:01 -0400 Subject: [PATCH 09/16] Harden replay wait invariants --- delivery/v1/replay.py | 132 +++++++++++--------- scripts/test/delivery-replay.test.sh | 177 +++++++++++++++++++++++---- 2 files changed, 233 insertions(+), 76 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 0b11b63..4e50663 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -15,6 +15,7 @@ import tempfile +LOADED_DRIVER_CODE = sys._getframe().f_code MAX_INPUT_BYTES = 8 * 1024 * 1024 MAX_OBSERVATION_BYTES = 64 * 1024 MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 @@ -150,8 +151,6 @@ def snapshot_file(source, destination, mode): def create_execution_snapshot(repository, arguments, state_dir): root = Path(tempfile.mkdtemp(prefix="execution-", dir=state_dir)) try: - driver = root / "delivery/v1/replay.py" - snapshot_file(Path(__file__).resolve(), driver, 0o500) core_relative = "scripts/core-contract.sh" core = snapshot_file(repository / core_relative, root / core_relative, 0o500) match = re.search( @@ -173,6 +172,17 @@ def create_execution_snapshot(repository, arguments, state_dir): raise +def driver_identity(): + source = read_bytes(trusted_file(Path(__file__).resolve()), MAX_INPUT_BYTES) + try: + current = compile(source, LOADED_DRIVER_CODE.co_filename, "exec") + except (SyntaxError, TypeError, ValueError) as error: + raise ReplayError("loaded replay driver identity is unavailable") from error + if current != LOADED_DRIVER_CODE: + raise ReplayError("loaded replay driver changed during startup") + return digest_bytes(source) + + def materializer_package_identity(repository): core_path = trusted_file(repository / "scripts/core-contract.sh") core_bytes = core_path.read_bytes() @@ -225,7 +235,7 @@ def input_identity(input_value, input_sha, arguments, execution): "path": safe_path(arguments.verify_path), "expected_sha256": expected, }, - "driver_sha256": digest_bytes(read_bytes(trusted_file(Path(__file__).resolve()), MAX_INPUT_BYTES)), + "driver_sha256": driver_identity(), "materializer_sha256": package["files"][PACKAGE_FILES[0]], "materializer_package": package, "closure_helper_sha256": digest_bytes(read_bytes( @@ -353,6 +363,18 @@ def verify_candidate(candidate_root, candidate_tree, path, expected): return actual +def revalidate_candidate(arguments, state): + expected = { + name: state["materialization"][name] + for name in ("candidate_commit_id", "candidate_tree_id", "candidate_parent_commit_id") + } + if candidate_identity(arguments.candidate_root, state["identity"]["source_commit_id"]) != expected: + raise ReplayError("candidate repository no longer matches saved materialization") + verify_candidate(arguments.candidate_root, expected["candidate_tree_id"], + state["identity"]["verifier"]["path"], + state["identity"]["verifier"]["expected_sha256"]) + + def observation(path, kind, identity, field): if path is None: return None @@ -453,12 +475,15 @@ def result(state): "state": state}, sort_keys=True, separators=(",", ":"))) -def replay(arguments): - execution = Path(arguments.execution_root).resolve() - if Path(__file__).resolve() != execution / "delivery/v1/replay.py": - raise ReplayError("execution snapshot is invalid") - state_dir = private_directory(arguments.state_dir) - disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) +def stop_if_interrupted(state, interrupted): + if not interrupted["value"]: + return False + if state is not None: + result(state) + return True + + +def replay_locked(arguments, execution, state_dir): state_path = state_dir / "run.json" input_snapshot_path = state_dir / "materialization-input.json" lock_path = state_dir / "replay.lock" @@ -482,6 +507,8 @@ def replay(arguments): if state is not None and any(state["identity"].get(name) != value for name, value in identity.items()): result({"phase": "stale", "reason": "run identity changed"}) return 2 + if stop_if_interrupted(state, interrupted): + return 75 if state is None: state = {"schema_version": 1, "kind": "delivery_replay_state", "identity": identity, "phase": "materializing", "authority": "none", "qualification": "unavailable"} @@ -498,14 +525,19 @@ def replay(arguments): result(state) return 1 if state["phase"] == "completed-offline": - verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], - identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) + revalidate_candidate(arguments, state) + if stop_if_interrupted(state, interrupted): + return 75 for supplied, kind, field, recorded in ( (arguments.review_observation, "delivery_replay_review_observation", "verdict", state.get("review")), (arguments.publisher_observation, "delivery_replay_publisher_observation", "disposition", state.get("publisher")), ): - if supplied is not None and observation(supplied, kind, state["identity"], field) != recorded: - raise ReplayError("supplied offline observation changed after completion") + if supplied is not None: + supplied_observation = observation(supplied, kind, state["identity"], field) + if stop_if_interrupted(state, interrupted): + return 75 + if supplied_observation != recorded: + raise ReplayError("supplied offline observation changed after completion") result(state) return 0 if state["phase"] == "materializing": @@ -526,8 +558,7 @@ def replay(arguments): }) state["phase"] = "verifying" atomic_json(state_path, state) - if interrupted["value"]: - result(state) + if stop_if_interrupted(state, interrupted): return 75 if state["phase"] == "verifying": try: @@ -541,13 +572,15 @@ def replay(arguments): return 1 state["phase"] = "review-wait" atomic_json(state_path, state) - if interrupted["value"]: - result(state) + if stop_if_interrupted(state, interrupted): return 75 if state["phase"] == "review-wait": - verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], - identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) + revalidate_candidate(arguments, state) + if stop_if_interrupted(state, interrupted): + return 75 review = observation(arguments.review_observation, "delivery_replay_review_observation", state["identity"], "verdict") + if stop_if_interrupted(state, interrupted): + return 75 if review is None: result(state) return 0 @@ -560,14 +593,20 @@ def replay(arguments): state["phase"] = "publish-wait" atomic_json(state_path, state) if state["phase"] == "publish-wait": - verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], - identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) - if arguments.review_observation is not None and ( - observation(arguments.review_observation, "delivery_replay_review_observation", - state["identity"], "verdict") != state.get("review") - ): - raise ReplayError("supplied offline review changed after review wait") + revalidate_candidate(arguments, state) + if stop_if_interrupted(state, interrupted): + return 75 + if arguments.review_observation is not None: + supplied_review = observation(arguments.review_observation, + "delivery_replay_review_observation", + state["identity"], "verdict") + if stop_if_interrupted(state, interrupted): + return 75 + if supplied_review != state.get("review"): + raise ReplayError("supplied offline review changed after review wait") publisher = observation(arguments.publisher_observation, "delivery_replay_publisher_observation", state["identity"], "disposition") + if stop_if_interrupted(state, interrupted): + return 75 if publisher is None: result(state) return 0 @@ -588,6 +627,17 @@ def replay(arguments): signal.signal(signal.SIGINT, previous_int) +def replay(arguments): + repository = Path(__file__).resolve().parents[2] + state_dir = private_directory(arguments.state_dir) + disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) + execution = create_execution_snapshot(repository, arguments, state_dir) + try: + return replay_locked(arguments, execution, state_dir) + finally: + shutil.rmtree(execution, ignore_errors=True) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("--input", required=True) @@ -602,36 +652,8 @@ def main(): parser.add_argument("--expected-sha256", required=True) parser.add_argument("--review-observation") parser.add_argument("--publisher-observation") - parser.add_argument("--execution-root", help=argparse.SUPPRESS) - arguments = parser.parse_args() try: - if arguments.execution_root is None: - state_dir = private_directory(arguments.state_dir) - execution = create_execution_snapshot(Path(__file__).resolve().parents[2], arguments, state_dir) - child = None - pending_signal = {"value": None} - - def forward_signal(number, _frame): - if child is None: - pending_signal["value"] = number - else: - child.send_signal(number) - - previous_term = signal.signal(signal.SIGTERM, forward_signal) - previous_int = signal.signal(signal.SIGINT, forward_signal) - try: - child = subprocess.Popen([ - sys.executable, str(execution / "delivery/v1/replay.py"), - *sys.argv[1:], "--execution-root", str(execution), - ]) - if pending_signal["value"] is not None: - child.send_signal(pending_signal["value"]) - return child.wait() - finally: - signal.signal(signal.SIGTERM, previous_term) - signal.signal(signal.SIGINT, previous_int) - shutil.rmtree(execution, ignore_errors=True) - return replay(arguments) + return replay(parser.parse_args()) except (OSError, ReplayError) as error: print(f"delivery replay: {error}", file=sys.stderr) return 1 diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 1b3f81f..8203504 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -114,6 +114,26 @@ jq -e '.state.phase=="review-wait" and .authority=="none" and .offline_simulatio fail missing-review-waits request_sha=$(jq -r '.identity.request_sha256' "$tmp/changed-state/run.json") candidate_tree=$(jq -r '.identity.candidate_tree_id' "$tmp/changed-state/run.json") +candidate_commit=$(jq -r '.identity.candidate_commit_id' "$tmp/changed-state/run.json") +moved_candidate=$(printf '%s\n' moved | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ + GIT_COMMITTER_EMAIL=fixture@example.invalid /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" \ + commit-tree "$candidate_tree" -p "$candidate_commit") +expect_candidate_move_rejected() { + local phase=$1 + shift + /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/candidate "$moved_candidate" + if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" "$@" >"$tmp/candidate-moved-$phase.out" 2>&1; then + /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/candidate "$candidate_commit" + fail "candidate-moved-$phase" + fi + /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/candidate "$candidate_commit" + grep -Fq 'candidate repository no longer matches saved materialization' "$tmp/candidate-moved-$phase.out" || + fail "candidate-moved-$phase-error" +} pass 'changed materialization and fixed read-only verification wait for review' cp "$base_input" "$tmp/mutable-input.json" @@ -147,26 +167,19 @@ pass 'replacement of the original input after snapshot cannot change materializa kill_wrapper="$tmp/kill-after-materialize.py" printf '%s\n' \ - 'import importlib.util, os, pathlib, signal, sys, types' \ + 'import importlib.util, os, signal, sys' \ 'path, arguments = sys.argv[1], sys.argv[2:]' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ 'spec.loader.exec_module(module)' \ - 'argument = lambda name: arguments[arguments.index(name) + 1]' \ - 'values = types.SimpleNamespace(closure_helper=argument("--closure-helper"), jq_bin=argument("--jq-bin"))' \ - 'snapshot = module.create_execution_snapshot(pathlib.Path(path).resolve().parents[2], values, pathlib.Path(argument("--state-dir")))' \ - 'snapshot_path = snapshot / "delivery/v1/replay.py"' \ - 'snapshot_spec = importlib.util.spec_from_file_location("snapshot_replay", snapshot_path)' \ - 'snapshot_module = importlib.util.module_from_spec(snapshot_spec)' \ - 'snapshot_spec.loader.exec_module(snapshot_module)' \ - 'original = snapshot_module.run_materializer' \ + 'original = module.run_materializer' \ 'def stop_after_materialization(*args, **kwargs):' \ ' result = original(*args, **kwargs)' \ ' os.kill(os.getpid(), signal.SIGKILL)' \ ' return result' \ - 'snapshot_module.run_materializer = stop_after_materialization' \ - 'sys.argv = [str(snapshot_path), *arguments, "--execution-root", str(snapshot)]' \ - 'raise SystemExit(snapshot_module.main())' >"$kill_wrapper" + 'module.run_materializer = stop_after_materialization' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$kill_wrapper" mkdir -m 700 "$tmp/reconcile-state" "$tmp/reconcile-candidate" "$tmp/reconcile-scratch" if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ @@ -233,9 +246,9 @@ if python3 "$replay" --input "$base_input" --source-repository-id fixture.target fail caller-execution-root fi [ "$(cat "$tmp/caller-execution-root/sentinel")" = keep ] || fail caller-execution-root-deleted -grep -Fq 'delivery replay: execution snapshot is invalid' "$tmp/caller-execution.out" || +grep -Fq 'unrecognized arguments: --execution-root' "$tmp/caller-execution.out" || fail caller-execution-root-error -pass 'a caller-supplied execution root is rejected without deleting it' +pass 'the replay CLI has no caller-selected execution-root path' huge_integer=$(printf '1%.0s' {1..5000}) printf '{"huge":%s}\n' "$huge_integer" >"$tmp/huge-input.json" @@ -302,6 +315,78 @@ pass 'invalid UTF-8 input, review, and journal records fail without a traceback' printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" +expect_candidate_move_rejected review-wait --review-observation "$tmp/review.json" +lock_holder="$tmp/lock-holder.py" +printf '%s\n' \ + 'import fcntl, pathlib, sys, time' \ + 'lock_path, ready, release = map(pathlib.Path, sys.argv[1:])' \ + 'with lock_path.open("a+b") as lock:' \ + ' fcntl.flock(lock, fcntl.LOCK_EX)' \ + ' ready.write_text("ready")' \ + ' while not release.exists(): time.sleep(0.01)' >"$lock_holder" +lock_wrapper="$tmp/lock-replay.py" +printf '%s\n' \ + 'import importlib.util, pathlib, sys' \ + 'path, marker, arguments = sys.argv[1], pathlib.Path(sys.argv[2]), sys.argv[3:]' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'original = module.fcntl.flock' \ + 'def marked_flock(*args, **kwargs):' \ + ' marker.write_text("waiting")' \ + ' return original(*args, **kwargs)' \ + 'module.fcntl.flock = marked_flock' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$lock_wrapper" +mkdir -m 700 "$tmp/lock-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/lock-state/" +python3 "$lock_holder" "$tmp/lock-state/replay.lock" "$tmp/lock-ready" "$tmp/lock-release" & +lock_holder_pid=$! +while [ ! -f "$tmp/lock-ready" ]; do sleep 0.01; done +python3 "$lock_wrapper" "$replay" "$tmp/lock-waiting" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/lock-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --review-observation "$tmp/review.json" >"$tmp/lock-cancel.out" & +lock_replay_pid=$! +while [ ! -f "$tmp/lock-waiting" ]; do sleep 0.01; done +kill -TERM "$lock_replay_pid" +touch "$tmp/lock-release" +wait "$lock_holder_pid" +if wait "$lock_replay_pid"; then fail lock-cancel-status; else lock_status=$?; fi +[ "$lock_status" -eq 75 ] || fail lock-cancel-code +jq -e '(.phase=="review-wait") and (has("review")|not)' "$tmp/lock-state/run.json" >/dev/null || + fail lock-cancel-state + +observation_wrapper="$tmp/observation-interrupt.py" +printf '%s\n' \ + 'import importlib.util, os, signal, sys' \ + 'path, signal_name, arguments = sys.argv[1], sys.argv[2], sys.argv[3:]' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'original = module.observation' \ + 'def interrupt_after_observation(*args, **kwargs):' \ + ' result = original(*args, **kwargs)' \ + ' os.kill(os.getpid(), getattr(signal, signal_name))' \ + ' return result' \ + 'module.observation = interrupt_after_observation' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$observation_wrapper" +mkdir -m 700 "$tmp/review-cancel-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/review-cancel-state/" +if python3 "$observation_wrapper" "$replay" SIGINT \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/review-cancel-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --review-observation "$tmp/review.json" >"$tmp/review-cancel.out"; then + fail review-cancel-status +else + review_cancel_status=$? +fi +[ "$review_cancel_status" -eq 75 ] || fail review-cancel-code +jq -e '(.phase=="review-wait") and (has("review")|not)' "$tmp/review-cancel-state/run.json" >/dev/null || + fail review-cancel-state printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/numeric-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ @@ -314,6 +399,22 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits +expect_candidate_move_rejected publish-wait --publisher-observation "$tmp/publisher.json" +mkdir -m 700 "$tmp/publish-cancel-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/publish-cancel-state/" +if python3 "$observation_wrapper" "$replay" SIGTERM \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/publish-cancel-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/publish-cancel.out"; then + fail publish-cancel-status +else + publish_cancel_status=$? +fi +[ "$publish_cancel_status" -eq 75 ] || fail publish-cancel-code +jq -e '(.phase=="publish-wait") and (has("publisher")|not)' "$tmp/publish-cancel-state/run.json" >/dev/null || + fail publish-cancel-state +pass 'SIGTERM at the lock and SIGINT or SIGTERM after wait observations do not advance state' printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/numeric-publisher.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ @@ -377,6 +478,8 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --review-observation "$tmp/review.json" --publisher-observation "$tmp/publisher.json" >"$tmp/completed.out" jq -e '.state.phase=="completed-offline" and .state.publisher.disposition=="offline-simulated"' "$tmp/completed.out" >/dev/null || fail completed-offline +expect_candidate_move_rejected completed-offline +pass 'review, publish, and completed waits reject a moved same-tree candidate ref' cp "$tmp/completed.out" "$tmp/completed-first.out" python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ @@ -520,33 +623,65 @@ pass 'changed executable package or replay driver cannot reuse a prior run' /bin/cp "$replay" "$package_replay" /bin/cp "$root/adapters/local-git-materializer/v1/protocol.jq" \ "$package_root/adapters/local-git-materializer/v1/protocol.jq" +printf '%s\n' keep >"$package_root/sentinel" +if python3 "$package_replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/package-candidate" --scratch-root "$tmp/package-scratch" --state-dir "$tmp/package-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --execution-root "$package_root" >"$tmp/copied-root-bypass.out" 2>&1; then + fail copied-root-bypass +fi +grep -Fq 'unrecognized arguments: --execution-root' "$tmp/copied-root-bypass.out" || fail copied-root-bypass-error +[ "$(cat "$package_root/sentinel")" = keep ] || fail copied-root-bypass-deleted +pass 'a copied expected layout cannot select or delete an execution root' + +driver_wrapper="$tmp/driver-identity.py" +printf '%s\n' \ + 'import hashlib, importlib.util, pathlib, sys' \ + 'driver = pathlib.Path(sys.argv[1])' \ + 'spec = importlib.util.spec_from_file_location("replay", driver)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'saved = driver.read_bytes()' \ + 'assert module.driver_identity() == hashlib.sha256(saved).hexdigest()' \ + 'for changed, expected in ((saved + b"\nCHANGED_EXECUTABLE_STATEMENT = True\n", "changed during startup"), (saved + b"\nif\n", "identity is unavailable"), (saved + b"\n\\xff\n", "identity is unavailable")):' \ + ' driver.write_bytes(changed)' \ + ' try:' \ + ' module.driver_identity()' \ + ' except module.ReplayError as error:' \ + ' assert expected in str(error)' \ + ' else:' \ + ' raise AssertionError("changed driver was accepted")' \ + 'driver.write_bytes(saved)' >"$driver_wrapper" +python3 "$driver_wrapper" "$package_replay" || fail driver-loaded-identity +pass 'driver identity binds normal loaded code and rejects changed, invalid, or undecodable source' + /bin/cp "$runtime/object-closure" "$tmp/race-object-closure" /bin/cp "$jq_bin" "$tmp/race-jq" /bin/chmod 0555 "$tmp/race-object-closure" "$tmp/race-jq" race_wrapper="$tmp/snapshot-race.py" printf '%s\n' \ - 'import argparse, importlib.util, os, pathlib, stat, subprocess, sys' \ + 'import argparse, importlib.util, pathlib, shutil, stat, sys' \ 'driver, repository, state_dir, helper, jq_bin, *arguments = sys.argv[1:]' \ 'spec = importlib.util.spec_from_file_location("replay", driver)' \ 'module = importlib.util.module_from_spec(spec)' \ 'spec.loader.exec_module(module)' \ - 'values = argparse.Namespace(closure_helper=helper, jq_bin=jq_bin)' \ + 'value = lambda name: arguments[arguments.index(name) + 1]' \ + 'values = argparse.Namespace(input=value("--input"), source_repository_id=value("--source-repository-id"), source_git_dir=value("--source-git-dir"), candidate_root=value("--candidate-root"), scratch_root=value("--scratch-root"), state_dir=value("--state-dir"), closure_helper=helper, jq_bin=jq_bin, verify_path=value("--verify-path"), expected_sha256=value("--expected-sha256"), review_observation=None, publisher_observation=None)' \ 'snapshot = module.create_execution_snapshot(pathlib.Path(repository), values, pathlib.Path(state_dir))' \ - 'targets = [pathlib.Path(driver), pathlib.Path(repository) / "adapters/local-git-materializer/v1/protocol.jq", pathlib.Path(helper), pathlib.Path(jq_bin)]' \ + 'targets = [pathlib.Path(repository) / "adapters/local-git-materializer/v1/protocol.jq", pathlib.Path(helper), pathlib.Path(jq_bin)]' \ 'saved = [target.read_bytes() for target in targets]' \ 'modes = [stat.S_IMODE(target.stat().st_mode) for target in targets]' \ 'try:' \ ' for target in targets:' \ ' target.chmod(0o700)' \ ' target.write_bytes(b"replaced after execution snapshot\n")' \ - ' result = subprocess.run([sys.executable, str(snapshot / "delivery/v1/replay.py"), *arguments, "--execution-root", str(snapshot)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)' \ + ' status = module.replay_locked(values, snapshot, pathlib.Path(state_dir))' \ 'finally:' \ ' for target, data, mode in zip(targets, saved, modes):' \ ' target.write_bytes(data)' \ ' target.chmod(mode)' \ - 'sys.stdout.buffer.write(result.stdout)' \ - 'sys.stderr.buffer.write(result.stderr)' \ - 'raise SystemExit(result.returncode)' >"$race_wrapper" + ' shutil.rmtree(snapshot, ignore_errors=True)' \ + 'raise SystemExit(status)' >"$race_wrapper" /bin/mkdir -m 700 "$tmp/race-state" "$tmp/race-candidate" "$tmp/race-scratch" driver_sha=$(sha_file "$package_replay") package_sha=$(sha_file "$package_root/adapters/local-git-materializer/v1/protocol.jq") From 1fa6e090eccebaa795e05f121b66da9d1ad09466 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 09:15:58 -0400 Subject: [PATCH 10/16] Preserve replay cancellation identity --- delivery/v1/replay.py | 21 +++++- scripts/test/delivery-replay.test.sh | 106 +++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 4e50663..1578c80 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -148,6 +148,13 @@ def snapshot_file(source, destination, mode): return data +def snapshot_native_executable(source, destination): + data = snapshot_file(source, destination, 0o500) + if not data.startswith((b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", + b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca")): + raise ReplayError("dependency is not a native executable") + + def create_execution_snapshot(repository, arguments, state_dir): root = Path(tempfile.mkdtemp(prefix="execution-", dir=state_dir)) try: @@ -164,8 +171,8 @@ def create_execution_snapshot(repository, arguments, state_dir): continue mode = 0o500 if relative.endswith(".sh") else 0o400 snapshot_file(repository / relative, root / relative, mode) - snapshot_file(arguments.closure_helper, root / ".dependencies/object-closure", 0o500) - snapshot_file(arguments.jq_bin, root / ".dependencies/jq", 0o500) + snapshot_native_executable(arguments.closure_helper, root / ".dependencies/object-closure") + snapshot_native_executable(arguments.jq_bin, root / ".dependencies/jq") return root except (OSError, ReplayError): shutil.rmtree(root, ignore_errors=True) @@ -218,8 +225,12 @@ def input_identity(input_value, input_sha, arguments, execution): raise ReplayError("materialization input request identity is invalid") if not isinstance(source_tree_id, str) or not OID.fullmatch(source_tree_id): raise ReplayError("materialization input tree identity is invalid") - if not isinstance(source, dict) or not OID.fullmatch(str(source.get("commit_id", ""))): + if not isinstance(source, dict) or not isinstance(source.get("commit_id"), str) or \ + not OID.fullmatch(source["commit_id"]): raise ReplayError("materialization input commit identity is invalid") + if not isinstance(source.get("hash_algorithm"), str) or \ + source["hash_algorithm"] not in {"sha1", "sha256"}: + raise ReplayError("materialization input hash algorithm is invalid") expected = arguments.expected_sha256 if not re.fullmatch(r"[0-9a-f]{64}", expected): raise ReplayError("expected verifier digest is invalid") @@ -548,6 +559,8 @@ def replay_locked(arguments, execution, state_dir): arguments, execution, input_snapshot_path, identity ) except ReplayError as error: + if stop_if_interrupted(state, interrupted): + return 75 state.update({"phase": "failed", "recoverable": True, "reason": str(error)}) atomic_json(state_path, state) result(state) @@ -566,6 +579,8 @@ def replay_locked(arguments, execution, state_dir): "sha256": verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], identity["verifier"]["path"], identity["verifier"]["expected_sha256"])} except ReplayError as error: + if stop_if_interrupted(state, interrupted): + return 75 state.update({"phase": "failed", "recoverable": False, "reason": str(error)}) atomic_json(state_path, state) result(state) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 8203504..adf2b1d 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -31,11 +31,7 @@ jq_bin="${TMPDIR:-/tmp}/ystack-portable-core-jq16/$asset" runtime="$tmp/runtime" /bin/mkdir -m 700 "$runtime" "$tmp/home" -if [ "$platform" = Darwin:arm64 ]; then - printf '%s\n' '#!/bin/sh' "exec /usr/bin/arch -x86_64 '$jq_bin' \"\$@\"" > "$runtime/jq" -else - /bin/cp "$jq_bin" "$runtime/jq" -fi +/bin/cp "$jq_bin" "$runtime/jq" /bin/chmod 0555 "$runtime/jq" jq_bin="$runtime/jq" PATH="$runtime:/usr/bin:/bin" @@ -109,6 +105,20 @@ run_replay() { --verify-path source.txt --expected-sha256 "$expected" } +printf '%s\n' '#!/bin/sh' "exec '$jq_bin' \"\$@\"" >"$tmp/jq-launcher" +/bin/chmod 0555 "$tmp/jq-launcher" +/bin/mkdir -m 700 "$tmp/launcher-state" "$tmp/launcher-candidate" "$tmp/launcher-scratch" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/launcher-candidate" --scratch-root "$tmp/launcher-scratch" \ + --state-dir "$tmp/launcher-state" --closure-helper "$runtime/object-closure" --jq-bin "$tmp/jq-launcher" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/launcher.out" 2>&1; then + fail dependency-launcher +fi +grep -Fq 'delivery replay: dependency is not a native executable' "$tmp/launcher.out" || + fail dependency-launcher-error +[ ! -e "$tmp/launcher-state/run.json" ] || fail dependency-launcher-journal +pass 'dependency launchers are rejected before replay state is created' + run_replay changed "$base_input" "$expected_changed" >"$tmp/changed.out" jq -e '.state.phase=="review-wait" and .authority=="none" and .offline_simulation==true' "$tmp/changed.out" >/dev/null || fail missing-review-waits @@ -212,6 +222,63 @@ jq -e '.state.phase=="failed" and (.state.reason|contains("does not match frozen fail reconcile-mismatch-state pass 'a mismatched interrupted candidate is rejected without cleanup' +group_interrupt_wrapper="$tmp/materialization-group-interrupt.py" +printf '%s\n' \ + 'import importlib.util, os, signal, sys' \ + 'path, point, signal_name, arguments = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4:]' \ + 'os.setsid()' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'original = module.run_materializer' \ + 'def interrupt_materialization(*args, **kwargs):' \ + ' if point == "before":' \ + ' os.killpg(os.getpgrp(), getattr(signal, signal_name))' \ + ' raise module.ReplayError("materialization did not complete")' \ + ' original(*args, **kwargs)' \ + ' os.killpg(os.getpgrp(), getattr(signal, signal_name))' \ + ' raise module.ReplayError("materialization did not complete")' \ + 'def interrupt_verification(*_args, **_kwargs):' \ + ' os.killpg(os.getpgrp(), getattr(signal, signal_name))' \ + ' raise module.ReplayError("fixed verifier could not read the candidate blob")' \ + 'if point == "verify":' \ + ' module.verify_candidate = interrupt_verification' \ + 'else:' \ + ' module.run_materializer = interrupt_materialization' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$group_interrupt_wrapper" +for interrupt_case in before after verify; do + if [ "$interrupt_case" = after ]; then interrupt_signal=SIGTERM; else interrupt_signal=SIGINT; fi + mkdir -m 700 "$tmp/group-$interrupt_case-state" "$tmp/group-$interrupt_case-candidate" \ + "$tmp/group-$interrupt_case-scratch" + if python3 "$group_interrupt_wrapper" "$replay" "$interrupt_case" "$interrupt_signal" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/group-$interrupt_case-candidate" --scratch-root "$tmp/group-$interrupt_case-scratch" \ + --state-dir "$tmp/group-$interrupt_case-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/group-$interrupt_case.out"; then + fail "group-$interrupt_case-status" + else + interrupt_status=$? + fi + [ "$interrupt_status" -eq 75 ] || fail "group-$interrupt_case-code" + if [ "$interrupt_case" = before ]; then + expected_interrupt_phase=materializing + [ ! -e "$tmp/group-$interrupt_case-candidate/repository.git" ] || fail group-before-effect + else + if [ "$interrupt_case" = after ]; then expected_interrupt_phase=materializing; else expected_interrupt_phase=verifying; fi + [ -d "$tmp/group-$interrupt_case-candidate/repository.git" ] || fail group-after-candidate + fi + jq -e --arg phase "$expected_interrupt_phase" '.phase==$phase' \ + "$tmp/group-$interrupt_case-state/run.json" >/dev/null || fail "group-$interrupt_case-state" + python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/group-$interrupt_case-candidate" --scratch-root "$tmp/group-$interrupt_case-scratch" \ + --state-dir "$tmp/group-$interrupt_case-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/group-$interrupt_case-resume.out" + jq -e '.state.phase=="review-wait"' "$tmp/group-$interrupt_case-resume.out" >/dev/null || + fail "group-$interrupt_case-resume" +done +pass 'process-group cancellation during materialization or verification resumes the same attempt' + for tree_case in numeric list null; do case "$tree_case" in numeric) tree_value=123 ;; @@ -235,6 +302,35 @@ for tree_case in numeric list null; do done pass 'non-string source tree identities fail without a traceback' +for identity_field in commit_id hash_algorithm; do + case "$identity_field" in + commit_id) identity_error='materialization input commit identity is invalid' ;; + hash_algorithm) identity_error='materialization input hash algorithm is invalid' ;; + esac + for identity_case in numeric list null; do + case "$identity_case" in + numeric) identity_value=1111111111111111111111111111111111111111 ;; + list) identity_value='[]' ;; + null) identity_value=null ;; + esac + "$jq_bin" -S -c ".stage_request.content.body.target_revision.value.$identity_field = $identity_value" \ + "$base_input" >"$tmp/$identity_field-$identity_case.json" + mkdir -m 700 "$tmp/$identity_field-$identity_case-state" "$tmp/$identity_field-$identity_case-candidate" \ + "$tmp/$identity_field-$identity_case-scratch" + if python3 "$replay" --input "$tmp/$identity_field-$identity_case.json" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/$identity_field-$identity_case-candidate" \ + --scratch-root "$tmp/$identity_field-$identity_case-scratch" --state-dir "$tmp/$identity_field-$identity_case-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" >"$tmp/$identity_field-$identity_case.out" 2>&1; then + fail "$identity_field-$identity_case" + fi + [ ! -e "$tmp/$identity_field-$identity_case-state/run.json" ] || fail "$identity_field-$identity_case-journal" + grep -Fq "delivery replay: $identity_error" \ + "$tmp/$identity_field-$identity_case.out" || fail "$identity_field-$identity_case-error" + done +done +pass 'non-string commit and hash-algorithm identities are rejected before journaling' + mkdir -m 700 "$tmp/caller-execution-root" "$tmp/caller-execution-state" \ "$tmp/caller-execution-candidate" "$tmp/caller-execution-scratch" printf '%s\n' keep >"$tmp/caller-execution-root/sentinel" From cc2cb987bbdd39ec62a766e6ff632f9f94b90af6 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 10:29:45 -0400 Subject: [PATCH 11/16] Bind replay completion atomically --- delivery/v1/replay.py | 182 ++++++++++++++++++++------- scripts/test/delivery-replay.test.sh | 150 ++++++++++++++++++---- 2 files changed, 259 insertions(+), 73 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 1578c80..86e7d8a 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -2,6 +2,7 @@ """Run one inactive, offline delivery replay without executing candidate code.""" import argparse +from contextlib import contextmanager import fcntl import hashlib import json @@ -13,9 +14,25 @@ import subprocess import sys import tempfile +import time -LOADED_DRIVER_CODE = sys._getframe().f_code +LOADED_DRIVER_BYTES = globals().get("_REPLAY_DRIVER_BYTES") +if LOADED_DRIVER_BYTES is None and __name__ == "__main__": + try: + driver_descriptor = os.open(__file__, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + with os.fdopen(driver_descriptor, "rb") as driver_handle: + driver_source = driver_handle.read(8 * 1024 * 1024 + 1) + if len(driver_source) > 8 * 1024 * 1024: + raise OSError("replay driver exceeds its size limit") + driver_code = compile(driver_source, __file__, "exec") + except (OSError, SyntaxError, TypeError, ValueError) as error: + print(f"delivery replay: loaded replay driver identity is unavailable: {error}", file=sys.stderr) + raise SystemExit(1) from error + globals()["_REPLAY_DRIVER_BYTES"] = driver_source + exec(driver_code, globals()) + raise SystemExit(1) + MAX_INPUT_BYTES = 8 * 1024 * 1024 MAX_OBSERVATION_BYTES = 64 * 1024 MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 @@ -149,14 +166,22 @@ def snapshot_file(source, destination, mode): def snapshot_native_executable(source, destination): - data = snapshot_file(source, destination, 0o500) + data = read_bytes(trusted_file(source), MAX_INPUT_BYTES) if not data.startswith((b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca")): raise ReplayError("dependency is not a native executable") + destination.parent.mkdir(parents=True, exist_ok=True) + atomic_bytes(destination, data) + os.chmod(destination, 0o500) def create_execution_snapshot(repository, arguments, state_dir): - root = Path(tempfile.mkdtemp(prefix="execution-", dir=state_dir)) + root = state_dir / "execution" + if root.exists(): + if root.is_symlink() or not root.is_dir(): + raise ReplayError("execution bundle is unavailable") + return root + os.mkdir(root, 0o700) try: core_relative = "scripts/core-contract.sh" core = snapshot_file(repository / core_relative, root / core_relative, 0o500) @@ -175,19 +200,31 @@ def create_execution_snapshot(repository, arguments, state_dir): snapshot_native_executable(arguments.jq_bin, root / ".dependencies/jq") return root except (OSError, ReplayError): - shutil.rmtree(root, ignore_errors=True) raise -def driver_identity(): - source = read_bytes(trusted_file(Path(__file__).resolve()), MAX_INPUT_BYTES) +def execution_sources_match(repository, arguments, execution): try: - current = compile(source, LOADED_DRIVER_CODE.co_filename, "exec") - except (SyntaxError, TypeError, ValueError) as error: - raise ReplayError("loaded replay driver identity is unavailable") from error - if current != LOADED_DRIVER_CODE: - raise ReplayError("loaded replay driver changed during startup") - return digest_bytes(source) + if materializer_package_identity(repository) != materializer_package_identity(execution): + return False + for source, saved in ( + (arguments.closure_helper, execution / ".dependencies/object-closure"), + (arguments.jq_bin, execution / ".dependencies/jq"), + ): + current = read_bytes(trusted_file(source), MAX_INPUT_BYTES) + if not current.startswith((b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", + b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca")) or \ + digest_bytes(current) != digest_bytes(read_bytes(trusted_file(saved), MAX_INPUT_BYTES)): + return False + return True + except (OSError, ReplayError): + return False + + +def driver_identity(): + if not isinstance(LOADED_DRIVER_BYTES, bytes) or len(LOADED_DRIVER_BYTES) > MAX_INPUT_BYTES: + raise ReplayError("loaded replay driver identity is unavailable") + return digest_bytes(LOADED_DRIVER_BYTES) def materializer_package_identity(repository): @@ -331,6 +368,50 @@ def candidate_identity(candidate_root, source_commit): "candidate_parent_commit_id": parent} +@contextmanager +def hold_candidate_ref(candidate_root, expected_commit): + repository = Path(candidate_root).resolve() / "repository.git" + lock_path = repository / "refs/heads/candidate.lock" + command = ["/usr/bin/git", f"--git-dir={repository}", "-c", "core.hooksPath=/dev/null", + "update-ref", "--stdin"] + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE) + try: + process.stdin.write( + f"option no-deref\nstart\nverify refs/heads/candidate {expected_commit}\nprepare\n".encode() + ) + process.stdin.flush() + for _ in range(1000): + if lock_path.is_file() and not lock_path.is_symlink(): + break + if process.poll() is not None: + raise ReplayError("candidate repository identity guard failed") + time.sleep(0.001) + else: + raise ReplayError("candidate repository identity guard timed out") + symbolic = subprocess.run( + ["/usr/bin/git", f"--git-dir={repository}", "symbolic-ref", "-q", "refs/heads/candidate"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False + ) + if symbolic.returncode != 1: + raise ReplayError("candidate repository identity guard failed") + yield + finally: + if process.poll() is None: + try: + process.stdin.write(b"abort\n") + process.stdin.flush() + except (BrokenPipeError, OSError): + pass + if process.stdin is not None: + process.stdin.close() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.terminate() + process.wait(timeout=5) + + def reconcile_materialization(arguments, execution, input_path, identity, state_dir): existing = candidate_identity(arguments.candidate_root, identity["source_commit_id"]) if existing is None: @@ -483,7 +564,7 @@ def validate_state(state, identity): def result(state): print(json.dumps({"kind": "delivery_replay_receipt", "authority": "none", "qualification": "unavailable", "offline_simulation": True, - "state": state}, sort_keys=True, separators=(",", ":"))) + "state": state}, sort_keys=True, separators=(",", ":")), flush=True) def stop_if_interrupted(state, interrupted): @@ -494,14 +575,11 @@ def stop_if_interrupted(state, interrupted): return True -def replay_locked(arguments, execution, state_dir): +def replay_locked(arguments, state_dir): + repository = Path(__file__).resolve().parents[2] state_path = state_dir / "run.json" input_snapshot_path = state_dir / "materialization-input.json" lock_path = state_dir / "replay.lock" - input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) - input_value = parse_json(input_bytes) - input_sha = digest_bytes(input_bytes) - identity = input_identity(input_value, input_sha, arguments, execution) interrupted = {"value": False} previous_term = signal.getsignal(signal.SIGTERM) previous_int = signal.getsignal(signal.SIGINT) @@ -511,6 +589,12 @@ def replay_locked(arguments, execution, state_dir): lock_descriptor = os.open(lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) with os.fdopen(lock_descriptor, "a+b") as lock: fcntl.flock(lock, fcntl.LOCK_EX) + execution = create_execution_snapshot(repository, arguments, state_dir) + sources_match = execution_sources_match(repository, arguments, execution) + input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) + input_value = parse_json(input_bytes) + input_sha = digest_bytes(input_bytes) + identity = input_identity(input_value, input_sha, arguments, execution) state = None if state_path.exists(): state = parse_json(read_bytes(state_path, MAX_OBSERVATION_BYTES)) @@ -518,6 +602,11 @@ def replay_locked(arguments, execution, state_dir): if state is not None and any(state["identity"].get(name) != value for name, value in identity.items()): result({"phase": "stale", "reason": "run identity changed"}) return 2 + if not sources_match: + if state is not None: + result({"phase": "stale", "reason": "execution dependencies changed"}) + return 2 + raise ReplayError("execution bundle does not match current dependencies") if stop_if_interrupted(state, interrupted): return 75 if state is None: @@ -608,33 +697,37 @@ def replay_locked(arguments, execution, state_dir): state["phase"] = "publish-wait" atomic_json(state_path, state) if state["phase"] == "publish-wait": - revalidate_candidate(arguments, state) - if stop_if_interrupted(state, interrupted): - return 75 - if arguments.review_observation is not None: - supplied_review = observation(arguments.review_observation, - "delivery_replay_review_observation", - state["identity"], "verdict") + with hold_candidate_ref(arguments.candidate_root, + state["materialization"]["candidate_commit_id"]): + revalidate_candidate(arguments, state) if stop_if_interrupted(state, interrupted): return 75 - if supplied_review != state.get("review"): - raise ReplayError("supplied offline review changed after review wait") - publisher = observation(arguments.publisher_observation, "delivery_replay_publisher_observation", state["identity"], "disposition") - if stop_if_interrupted(state, interrupted): - return 75 - if publisher is None: - result(state) - return 0 - if publisher["disposition"] != "offline-simulated": - state.update({"phase": "failed", "recoverable": False, "reason": "offline publisher disposition is invalid"}) + if arguments.review_observation is not None: + supplied_review = observation(arguments.review_observation, + "delivery_replay_review_observation", + state["identity"], "verdict") + if stop_if_interrupted(state, interrupted): + return 75 + if supplied_review != state.get("review"): + raise ReplayError("supplied offline review changed after review wait") + publisher = observation(arguments.publisher_observation, + "delivery_replay_publisher_observation", + state["identity"], "disposition") + if stop_if_interrupted(state, interrupted): + return 75 + if publisher is None: + result(state) + return 0 + if publisher["disposition"] != "offline-simulated": + state.update({"phase": "failed", "recoverable": False, "reason": "offline publisher disposition is invalid"}) + atomic_json(state_path, state) + result(state) + return 1 + state["publisher"] = publisher + state["phase"] = "completed-offline" atomic_json(state_path, state) result(state) - return 1 - state["publisher"] = publisher - state["phase"] = "completed-offline" - atomic_json(state_path, state) - result(state) - return 0 + return 0 result(state) return 1 finally: @@ -643,14 +736,9 @@ def replay_locked(arguments, execution, state_dir): def replay(arguments): - repository = Path(__file__).resolve().parents[2] state_dir = private_directory(arguments.state_dir) disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) - execution = create_execution_snapshot(repository, arguments, state_dir) - try: - return replay_locked(arguments, execution, state_dir) - finally: - shutil.rmtree(execution, ignore_errors=True) + return replay_locked(arguments, state_dir) def main(): diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index adf2b1d..f67d178 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -141,7 +141,8 @@ expect_candidate_move_rejected() { fail "candidate-moved-$phase" fi /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/candidate "$candidate_commit" - grep -Fq 'candidate repository no longer matches saved materialization' "$tmp/candidate-moved-$phase.out" || + grep -Eq 'candidate repository no longer matches saved materialization|candidate repository identity guard failed' \ + "$tmp/candidate-moved-$phase.out" || fail "candidate-moved-$phase-error" } pass 'changed materialization and fixed read-only verification wait for review' @@ -177,11 +178,12 @@ pass 'replacement of the original input after snapshot cannot change materializa kill_wrapper="$tmp/kill-after-materialize.py" printf '%s\n' \ - 'import importlib.util, os, signal, sys' \ + 'import importlib.util, os, pathlib, signal, sys' \ 'path, arguments = sys.argv[1], sys.argv[2:]' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ 'original = module.run_materializer' \ 'def stop_after_materialization(*args, **kwargs):' \ ' result = original(*args, **kwargs)' \ @@ -204,6 +206,25 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- jq -e '.state.phase=="review-wait"' "$tmp/reconcile-retry.out" >/dev/null || fail reconcile-retry pass 'SIGKILL after materializer output reconciles the existing candidate once' +mkdir -m 700 "$tmp/repeated-kill-state" "$tmp/repeated-kill-candidate" "$tmp/repeated-kill-scratch" +for kill_round in 1 2 3; do + if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/repeated-kill-candidate" \ + --scratch-root "$tmp/repeated-kill-scratch" --state-dir "$tmp/repeated-kill-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" >"$tmp/repeated-kill-$kill_round.out" 2>&1; then + fail "repeated-kill-$kill_round" + fi + [ "$(find "$tmp/repeated-kill-state" -maxdepth 1 -type d -name 'execution*' | wc -l | tr -d ' ')" = 1 ] || + fail "repeated-kill-snapshot-count-$kill_round" +done +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/repeated-kill-candidate" --scratch-root "$tmp/repeated-kill-scratch" \ + --state-dir "$tmp/repeated-kill-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/repeated-kill-resume.out" +jq -e '.state.phase=="review-wait"' "$tmp/repeated-kill-resume.out" >/dev/null || fail repeated-kill-resume +pass 'repeated SIGKILL recovery reuses one bounded execution bundle' + mkdir -m 700 "$tmp/reconcile-bad-state" "$tmp/reconcile-bad-candidate" "$tmp/reconcile-bad-scratch" if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/reconcile-bad-candidate" --scratch-root "$tmp/reconcile-bad-scratch" --state-dir "$tmp/reconcile-bad-state" \ @@ -224,12 +245,13 @@ pass 'a mismatched interrupted candidate is rejected without cleanup' group_interrupt_wrapper="$tmp/materialization-group-interrupt.py" printf '%s\n' \ - 'import importlib.util, os, signal, sys' \ + 'import importlib.util, os, pathlib, signal, sys' \ 'path, point, signal_name, arguments = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4:]' \ 'os.setsid()' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ 'original = module.run_materializer' \ 'def interrupt_materialization(*args, **kwargs):' \ ' if point == "before":' \ @@ -426,7 +448,8 @@ printf '%s\n' \ 'path, marker, arguments = sys.argv[1], pathlib.Path(sys.argv[2]), sys.argv[3:]' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ 'original = module.fcntl.flock' \ 'def marked_flock(*args, **kwargs):' \ ' marker.write_text("waiting")' \ @@ -456,11 +479,12 @@ jq -e '(.phase=="review-wait") and (has("review")|not)' "$tmp/lock-state/run.jso observation_wrapper="$tmp/observation-interrupt.py" printf '%s\n' \ - 'import importlib.util, os, signal, sys' \ + 'import importlib.util, os, pathlib, signal, sys' \ 'path, signal_name, arguments = sys.argv[1], sys.argv[2], sys.argv[3:]' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ 'original = module.observation' \ 'def interrupt_after_observation(*args, **kwargs):' \ ' result = original(*args, **kwargs)' \ @@ -496,6 +520,19 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits expect_candidate_move_rejected publish-wait --publisher-observation "$tmp/publisher.json" +git_clean --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/alternate "$candidate_commit" +git_clean --git-dir="$tmp/changed-candidate/repository.git" symbolic-ref refs/heads/candidate refs/heads/alternate +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/candidate-symref.out" 2>&1; then + fail candidate-symref +fi +git_clean --git-dir="$tmp/changed-candidate/repository.git" symbolic-ref --delete refs/heads/candidate +git_clean --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/candidate "$candidate_commit" +git_clean --git-dir="$tmp/changed-candidate/repository.git" update-ref -d refs/heads/alternate +grep -Fq 'candidate repository identity guard failed' "$tmp/candidate-symref.out" || fail candidate-symref-error +pass 'candidate completion guard rejects a symbolic candidate ref' mkdir -m 700 "$tmp/publish-cancel-state" cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/publish-cancel-state/" if python3 "$observation_wrapper" "$replay" SIGTERM \ @@ -511,6 +548,63 @@ fi jq -e '(.phase=="publish-wait") and (has("publisher")|not)' "$tmp/publish-cancel-state/run.json" >/dev/null || fail publish-cancel-state pass 'SIGTERM at the lock and SIGINT or SIGTERM after wait observations do not advance state' + +atomic_wrapper="$tmp/atomic-candidate-observation.py" +printf '%s\n' \ + 'import importlib.util, os, pathlib, signal, subprocess, sys' \ + 'path, mode, marker, moved, arguments = sys.argv[1], sys.argv[2], pathlib.Path(sys.argv[3]), sys.argv[4], sys.argv[5:]' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ + 'original = module.observation' \ + 'def act_during_publisher(*args, **kwargs):' \ + ' result = original(*args, **kwargs)' \ + ' if args[1] == "delivery_replay_publisher_observation":' \ + ' if mode == "move":' \ + ' candidate = pathlib.Path(arguments[arguments.index("--candidate-root") + 1]) / "repository.git"' \ + ' attempt = subprocess.run(["/usr/bin/git", f"--git-dir={candidate}", "-c", "core.hooksPath=/dev/null", "update-ref", "refs/heads/candidate", moved], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)' \ + ' marker.write_text(str(attempt.returncode))' \ + ' else:' \ + ' os.kill(os.getpid(), signal.SIGKILL)' \ + ' return result' \ + 'module.observation = act_during_publisher' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$atomic_wrapper" +mkdir -m 700 "$tmp/atomic-move-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/atomic-move-state/" +python3 "$atomic_wrapper" "$replay" move "$tmp/atomic-move-status" "$moved_candidate" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/atomic-move-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/atomic-move.out" +jq -e '.state.phase=="completed-offline"' "$tmp/atomic-move.out" >/dev/null || fail atomic-move-completion +[ "$(cat "$tmp/atomic-move-status")" != 0 ] || fail atomic-move-lock +[ "$(git_clean --git-dir="$tmp/changed-candidate/repository.git" rev-parse refs/heads/candidate)" = "$candidate_commit" ] || + fail atomic-move-ref + +mkdir -m 700 "$tmp/atomic-kill-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/atomic-kill-state/" +if python3 "$atomic_wrapper" "$replay" kill "$tmp/unused-kill-status" "$moved_candidate" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/atomic-kill-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/atomic-kill.out" 2>&1; then + fail atomic-kill-status +fi +atomic_lock="$tmp/changed-candidate/repository.git/refs/heads/candidate.lock" +atomic_wait=0 +while [ -e "$atomic_lock" ]; do + atomic_wait=$((atomic_wait + 1)) + [ "$atomic_wait" -le 100 ] || fail atomic-kill-lock-release + sleep 0.01 +done +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/atomic-kill-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/atomic-kill-resume.out" +jq -e '.state.phase=="completed-offline"' "$tmp/atomic-kill-resume.out" >/dev/null || fail atomic-kill-resume +pass 'candidate ref guard blocks publisher-time moves and releases after SIGKILL' printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/numeric-publisher.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ @@ -734,36 +828,33 @@ driver_wrapper="$tmp/driver-identity.py" printf '%s\n' \ 'import hashlib, importlib.util, pathlib, sys' \ 'driver = pathlib.Path(sys.argv[1])' \ + 'saved = driver.read_bytes()' \ 'spec = importlib.util.spec_from_file_location("replay", driver)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ - 'saved = driver.read_bytes()' \ + 'module._REPLAY_DRIVER_BYTES = saved' \ + 'exec(compile(saved, str(driver), "exec"), module.__dict__)' \ 'assert module.driver_identity() == hashlib.sha256(saved).hexdigest()' \ - 'for changed, expected in ((saved + b"\nCHANGED_EXECUTABLE_STATEMENT = True\n", "changed during startup"), (saved + b"\nif\n", "identity is unavailable"), (saved + b"\n\\xff\n", "identity is unavailable")):' \ + 'for changed in (saved + b"\n# comment after load\n", saved + b"\n\n", saved + b"\nCHANGED_EXECUTABLE_STATEMENT = True\n", saved + b"\n\\xff\n"):' \ ' driver.write_bytes(changed)' \ - ' try:' \ - ' module.driver_identity()' \ - ' except module.ReplayError as error:' \ - ' assert expected in str(error)' \ - ' else:' \ - ' raise AssertionError("changed driver was accepted")' \ + ' assert module.driver_identity() == hashlib.sha256(saved).hexdigest()' \ 'driver.write_bytes(saved)' >"$driver_wrapper" python3 "$driver_wrapper" "$package_replay" || fail driver-loaded-identity -pass 'driver identity binds normal loaded code and rejects changed, invalid, or undecodable source' +pass 'driver identity remains bound to the exact source buffer loaded once' /bin/cp "$runtime/object-closure" "$tmp/race-object-closure" /bin/cp "$jq_bin" "$tmp/race-jq" /bin/chmod 0555 "$tmp/race-object-closure" "$tmp/race-jq" race_wrapper="$tmp/snapshot-race.py" printf '%s\n' \ - 'import argparse, importlib.util, pathlib, shutil, stat, sys' \ + 'import argparse, importlib.util, pathlib, stat, sys' \ 'driver, repository, state_dir, helper, jq_bin, *arguments = sys.argv[1:]' \ 'spec = importlib.util.spec_from_file_location("replay", driver)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(driver).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, driver, "exec"), module.__dict__)' \ 'value = lambda name: arguments[arguments.index(name) + 1]' \ 'values = argparse.Namespace(input=value("--input"), source_repository_id=value("--source-repository-id"), source_git_dir=value("--source-git-dir"), candidate_root=value("--candidate-root"), scratch_root=value("--scratch-root"), state_dir=value("--state-dir"), closure_helper=helper, jq_bin=jq_bin, verify_path=value("--verify-path"), expected_sha256=value("--expected-sha256"), review_observation=None, publisher_observation=None)' \ - 'snapshot = module.create_execution_snapshot(pathlib.Path(repository), values, pathlib.Path(state_dir))' \ + 'module.create_execution_snapshot(pathlib.Path(repository), values, pathlib.Path(state_dir))' \ 'targets = [pathlib.Path(repository) / "adapters/local-git-materializer/v1/protocol.jq", pathlib.Path(helper), pathlib.Path(jq_bin)]' \ 'saved = [target.read_bytes() for target in targets]' \ 'modes = [stat.S_IMODE(target.stat().st_mode) for target in targets]' \ @@ -771,13 +862,16 @@ printf '%s\n' \ ' for target in targets:' \ ' target.chmod(0o700)' \ ' target.write_bytes(b"replaced after execution snapshot\n")' \ - ' status = module.replay_locked(values, snapshot, pathlib.Path(state_dir))' \ + ' try:' \ + ' module.replay_locked(values, pathlib.Path(state_dir))' \ + ' except module.ReplayError as error:' \ + ' assert "execution bundle does not match current dependencies" in str(error)' \ + ' else:' \ + ' raise AssertionError("changed execution sources were accepted")' \ 'finally:' \ ' for target, data, mode in zip(targets, saved, modes):' \ ' target.write_bytes(data)' \ - ' target.chmod(mode)' \ - ' shutil.rmtree(snapshot, ignore_errors=True)' \ - 'raise SystemExit(status)' >"$race_wrapper" + ' target.chmod(mode)' >"$race_wrapper" /bin/mkdir -m 700 "$tmp/race-state" "$tmp/race-candidate" "$tmp/race-scratch" driver_sha=$(sha_file "$package_replay") package_sha=$(sha_file "$package_root/adapters/local-git-materializer/v1/protocol.jq") @@ -786,6 +880,10 @@ jq_sha=$(sha_file "$tmp/race-jq") python3 "$race_wrapper" "$package_replay" "$package_root" "$tmp/race-state" \ "$tmp/race-object-closure" "$tmp/race-jq" \ --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/race-candidate" --scratch-root "$tmp/race-scratch" --state-dir "$tmp/race-state" \ + --closure-helper "$tmp/race-object-closure" --jq-bin "$tmp/race-jq" \ + --verify-path source.txt --expected-sha256 "$expected_changed" +python3 "$package_replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/race-candidate" --scratch-root "$tmp/race-scratch" --state-dir "$tmp/race-state" \ --closure-helper "$tmp/race-object-closure" --jq-bin "$tmp/race-jq" \ --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/race.out" @@ -795,6 +893,6 @@ jq -e '.state.phase=="review-wait" and .state.identity.closure_helper_sha256==$helper and .state.identity.jq_sha256==$jq' \ --arg driver "$driver_sha" --arg package "$package_sha" --arg helper "$helper_sha" --arg jq "$jq_sha" \ "$tmp/race.out" >/dev/null || fail immutable-execution-snapshot -pass 'replacement after the private execution snapshot cannot change executed or recorded bytes' +pass 'the one state-owned execution bundle rejects drift and records its exact bytes' printf 'delivery replay: %s focused checks passed\n' "$passed" From 5f9c0d3fdcf57c1510a857fb7c49067fc55824d3 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 12:48:06 -0400 Subject: [PATCH 12/16] Rebuild incomplete execution bundles and isolate the ref guard Completes the in-progress fix for the two review findings on this PR. - The execution bundle is built in a program-owned staging directory and published by one rename, so an interrupted or rejected first attempt never leaves a usable-looking bundle. Foreign staging data and links are preserved, never deleted. A corrected native dependency can reuse the same state directory after a launcher rejection. - Every candidate Git operation, including the update-ref guard and the symbolic-ref check, runs under one sanitized environment, so ambient GIT_NAMESPACE, GIT_COMMON_DIR, config, work-tree, and hook settings cannot redirect or block the ref guard. - An existing bundle is judged by execution_sources_match, so changed or invalid dependencies read as a mismatch rather than a build failure. Three new focused checks cover each behavior; 33/33 pass. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/replay.py | 152 ++++++++++++++++----------- scripts/test/delivery-replay.test.sh | 95 ++++++++++++++++- 2 files changed, 187 insertions(+), 60 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 86e7d8a..22f246c 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -38,6 +38,15 @@ MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 OID = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}\Z") ACTOR = re.compile(r"[a-z0-9][a-z0-9._:-]{0,127}\Z") +GIT_ENVIRONMENT = { + "PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0", +} +NATIVE_EXECUTABLE_MAGICS = ( + b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", + b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca", +) PACKAGE_FILES = ( "adapters/local-git-materializer/v1/materialize.sh", "adapters/local-git-materializer/v1/protocol.jq", @@ -157,66 +166,97 @@ def package_paths(generation): return PACKAGE_FILES + tuple(f"{root}/{name}" for name in GENERATION_FILES) -def snapshot_file(source, destination, mode): - data = read_bytes(trusted_file(source), MAX_INPUT_BYTES) - destination.parent.mkdir(parents=True, exist_ok=True) - atomic_bytes(destination, data) - os.chmod(destination, mode) - return data +def execution_source_bytes(repository, arguments): + core_relative = "scripts/core-contract.sh" + core = read_bytes(trusted_file(repository / core_relative), MAX_INPUT_BYTES) + match = re.search( + rb"^PORTABLE_CORE_GENERATION='(g-[0-9a-f]{64})'$", core, re.MULTILINE + ) + if match is None: + raise ReplayError("materializer package generation is unavailable") + generation = match.group(1).decode() + package = { + relative: core if relative == core_relative else + read_bytes(trusted_file(repository / relative), MAX_INPUT_BYTES) + for relative in package_paths(generation) + } + dependencies = { + ".dependencies/object-closure": read_bytes( + trusted_file(arguments.closure_helper), MAX_INPUT_BYTES + ), + ".dependencies/jq": read_bytes(trusted_file(arguments.jq_bin), MAX_INPUT_BYTES), + } + if any(not data.startswith(NATIVE_EXECUTABLE_MAGICS) for data in dependencies.values()): + raise ReplayError("dependency is not a native executable") + return package | dependencies -def snapshot_native_executable(source, destination): - data = read_bytes(trusted_file(source), MAX_INPUT_BYTES) - if not data.startswith((b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", - b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca")): - raise ReplayError("dependency is not a native executable") - destination.parent.mkdir(parents=True, exist_ok=True) - atomic_bytes(destination, data) - os.chmod(destination, 0o500) +def owned_staging_token(owner_path): + if not owner_path.exists() or owner_path.is_symlink() or not owner_path.is_file(): + return None + value = read_bytes(owner_path, 128) + match = re.fullmatch(rb"ystack-delivery-execution-v1:([0-9a-f]{64})\n", value) + return match.group(1) if match is not None else None def create_execution_snapshot(repository, arguments, state_dir): root = state_dir / "execution" - if root.exists(): + staging = state_dir / ".execution-building" + owner_path = state_dir / ".execution-building.owner" + if root.is_symlink() or root.exists(): if root.is_symlink() or not root.is_dir(): raise ReplayError("execution bundle is unavailable") + # An existing bundle is judged by execution_sources_match, so changed or + # invalid dependencies read as a mismatch there, not as a build failure here. return root - os.mkdir(root, 0o700) + source_bytes = execution_source_bytes(repository, arguments) + token = owned_staging_token(owner_path) + if staging.is_symlink() or staging.exists(): + if staging.is_symlink() or not staging.is_dir() or token is None: + raise ReplayError("execution bundle staging is not program-owned") + marker = staging / ".owner" + entries = list(staging.iterdir()) + if entries and ( + marker not in entries or marker.is_symlink() or not marker.is_file() or + read_bytes(marker, 128) != token + b"\n" + ): + raise ReplayError("execution bundle staging is not program-owned") + shutil.rmtree(staging) + if token is None: + if owner_path.exists(): + raise ReplayError("execution bundle staging owner is invalid") + token = os.urandom(32).hex().encode() + descriptor = os.open(owner_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | + getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(descriptor, "wb") as owner: + owner.write(b"ystack-delivery-execution-v1:" + token + b"\n") + owner.flush() + os.fsync(owner.fileno()) + os.mkdir(staging, 0o700) + atomic_bytes(staging / ".owner", token + b"\n") + os.chmod(staging / ".owner", 0o400) + for relative, data in source_bytes.items(): + destination = staging / relative + destination.parent.mkdir(parents=True, exist_ok=True) + atomic_bytes(destination, data) + mode = 0o500 if relative.endswith(".sh") or relative.startswith(".dependencies/") else 0o400 + os.chmod(destination, mode) + os.replace(staging, root) + directory = os.open(state_dir, os.O_DIRECTORY) try: - core_relative = "scripts/core-contract.sh" - core = snapshot_file(repository / core_relative, root / core_relative, 0o500) - match = re.search( - rb"^PORTABLE_CORE_GENERATION='(g-[0-9a-f]{64})'$", core, re.MULTILINE - ) - if match is None: - raise ReplayError("materializer package generation is unavailable") - generation = match.group(1).decode() - for relative in package_paths(generation): - if relative == core_relative: - continue - mode = 0o500 if relative.endswith(".sh") else 0o400 - snapshot_file(repository / relative, root / relative, mode) - snapshot_native_executable(arguments.closure_helper, root / ".dependencies/object-closure") - snapshot_native_executable(arguments.jq_bin, root / ".dependencies/jq") - return root - except (OSError, ReplayError): - raise + os.fsync(directory) + finally: + os.close(directory) + os.unlink(owner_path) + return root def execution_sources_match(repository, arguments, execution): try: - if materializer_package_identity(repository) != materializer_package_identity(execution): - return False - for source, saved in ( - (arguments.closure_helper, execution / ".dependencies/object-closure"), - (arguments.jq_bin, execution / ".dependencies/jq"), - ): - current = read_bytes(trusted_file(source), MAX_INPUT_BYTES) - if not current.startswith((b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", - b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca")) or \ - digest_bytes(current) != digest_bytes(read_bytes(trusted_file(saved), MAX_INPUT_BYTES)): - return False - return True + return all( + digest_bytes(data) == digest_bytes(read_bytes(trusted_file(execution / relative), MAX_INPUT_BYTES)) + for relative, data in execution_source_bytes(repository, arguments).items() + ) except (OSError, ReplayError): return False @@ -345,13 +385,10 @@ def candidate_identity(candidate_root, source_commit): repository = Path(candidate_root).resolve() / "repository.git" if not repository.is_dir() or repository.is_symlink(): return None - environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", - "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", - "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} values = [] for revision in ("refs/heads/candidate", "refs/heads/candidate^{tree}"): result = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "rev-parse", revision], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + env=GIT_ENVIRONMENT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) value = result.stdout.decode().strip() if result.returncode != 0 or not OID.fullmatch(value): return None @@ -360,7 +397,7 @@ def candidate_identity(candidate_root, source_commit): return {"candidate_commit_id": values[0], "candidate_tree_id": values[1], "candidate_parent_commit_id": source_commit} result = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "rev-parse", "refs/heads/candidate^"], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + env=GIT_ENVIRONMENT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) parent = result.stdout.decode().strip() if result.returncode != 0 or not OID.fullmatch(parent): return None @@ -374,8 +411,8 @@ def hold_candidate_ref(candidate_root, expected_commit): lock_path = repository / "refs/heads/candidate.lock" command = ["/usr/bin/git", f"--git-dir={repository}", "-c", "core.hooksPath=/dev/null", "update-ref", "--stdin"] - process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE) + process = subprocess.Popen(command, env=GIT_ENVIRONMENT, stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) try: process.stdin.write( f"option no-deref\nstart\nverify refs/heads/candidate {expected_commit}\nprepare\n".encode() @@ -391,7 +428,7 @@ def hold_candidate_ref(candidate_root, expected_commit): raise ReplayError("candidate repository identity guard timed out") symbolic = subprocess.run( ["/usr/bin/git", f"--git-dir={repository}", "symbolic-ref", "-q", "refs/heads/candidate"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False + env=GIT_ENVIRONMENT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False ) if symbolic.returncode != 1: raise ReplayError("candidate repository identity guard failed") @@ -437,16 +474,13 @@ def verify_candidate(candidate_root, candidate_tree, path, expected): repository = Path(candidate_root).resolve() / "repository.git" if not repository.is_dir() or repository.is_symlink() or not OID.fullmatch(candidate_tree): raise ReplayError("candidate repository identity is unavailable") - environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", - "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", - "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} object_name = f"{candidate_tree}:{path}" size = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "cat-file", "-s", object_name], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + env=GIT_ENVIRONMENT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) if size.returncode != 0 or not size.stdout.strip().isdigit() or int(size.stdout) > MAX_VERIFIED_BLOB_BYTES: raise ReplayError("fixed verifier cannot read the candidate blob") blob = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "cat-file", "blob", object_name], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + env=GIT_ENVIRONMENT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) if blob.returncode != 0 or len(blob.stdout) != int(size.stdout): raise ReplayError("fixed verifier could not read the candidate blob") actual = digest_bytes(blob.stdout) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index f67d178..4ae7db5 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -117,7 +117,81 @@ fi grep -Fq 'delivery replay: dependency is not a native executable' "$tmp/launcher.out" || fail dependency-launcher-error [ ! -e "$tmp/launcher-state/run.json" ] || fail dependency-launcher-journal -pass 'dependency launchers are rejected before replay state is created' +python3 "$replay" --input "$base_input" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/launcher-candidate" --scratch-root "$tmp/launcher-scratch" \ + --state-dir "$tmp/launcher-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/launcher-retry.out" +jq -e '.state.phase=="review-wait"' "$tmp/launcher-retry.out" >/dev/null || fail dependency-launcher-retry +pass 'a corrected native dependency can reuse state after launcher rejection' + +snapshot_interrupt_wrapper="$tmp/snapshot-interrupt.py" +printf '%s\n' \ + 'import importlib.util, pathlib, sys' \ + 'path, point, arguments = sys.argv[1], sys.argv[2], sys.argv[3:]' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ + 'if point == "directory":' \ + ' original_mkdir = module.os.mkdir' \ + ' def interrupted_mkdir(target, *args, **kwargs):' \ + ' result = original_mkdir(target, *args, **kwargs)' \ + ' if pathlib.Path(target).name == ".execution-building": raise module.ReplayError("snapshot interrupted after directory creation")' \ + ' return result' \ + ' module.os.mkdir = interrupted_mkdir' \ + 'else:' \ + ' original_atomic = module.atomic_bytes' \ + ' writes = {"count": 0}' \ + ' def interrupted_atomic(target, data):' \ + ' original_atomic(target, data)' \ + ' if ".execution-building" in pathlib.Path(target).parts:' \ + ' writes["count"] += 1' \ + ' if writes["count"] == 2: raise module.ReplayError("snapshot interrupted during copy")' \ + ' module.atomic_bytes = interrupted_atomic' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$snapshot_interrupt_wrapper" +for snapshot_point in directory copy; do + mkdir -m 700 "$tmp/snapshot-$snapshot_point-state" "$tmp/snapshot-$snapshot_point-candidate" \ + "$tmp/snapshot-$snapshot_point-scratch" + if python3 "$snapshot_interrupt_wrapper" "$replay" "$snapshot_point" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/snapshot-$snapshot_point-candidate" --scratch-root "$tmp/snapshot-$snapshot_point-scratch" \ + --state-dir "$tmp/snapshot-$snapshot_point-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/snapshot-$snapshot_point.out" 2>&1; then + fail "snapshot-$snapshot_point-interrupt" + fi + [ ! -e "$tmp/snapshot-$snapshot_point-state/run.json" ] || fail "snapshot-$snapshot_point-journal" + [ -d "$tmp/snapshot-$snapshot_point-state/.execution-building" ] || fail "snapshot-$snapshot_point-staging" + python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/snapshot-$snapshot_point-candidate" --scratch-root "$tmp/snapshot-$snapshot_point-scratch" \ + --state-dir "$tmp/snapshot-$snapshot_point-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/snapshot-$snapshot_point-retry.out" + jq -e '.state.phase=="review-wait"' "$tmp/snapshot-$snapshot_point-retry.out" >/dev/null || + fail "snapshot-$snapshot_point-retry" +done + +mkdir -m 700 "$tmp/foreign-staging-state" "$tmp/foreign-staging-state/.execution-building" \ + "$tmp/foreign-staging-candidate" "$tmp/foreign-staging-scratch" +printf '%s\n' preserve >"$tmp/foreign-staging-state/.execution-building/sentinel" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/foreign-staging-candidate" --scratch-root "$tmp/foreign-staging-scratch" \ + --state-dir "$tmp/foreign-staging-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/foreign-staging.out" 2>&1; then + fail foreign-staging +fi +[ "$(cat "$tmp/foreign-staging-state/.execution-building/sentinel")" = preserve ] || fail foreign-staging-preserved +mkdir -m 700 "$tmp/foreign-link-state" "$tmp/foreign-link-target" "$tmp/foreign-link-candidate" "$tmp/foreign-link-scratch" +printf '%s\n' preserve >"$tmp/foreign-link-target/sentinel" +ln -s "$tmp/foreign-link-target" "$tmp/foreign-link-state/.execution-building" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/foreign-link-candidate" --scratch-root "$tmp/foreign-link-scratch" \ + --state-dir "$tmp/foreign-link-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/foreign-link.out" 2>&1; then + fail foreign-staging-link +fi +[ -L "$tmp/foreign-link-state/.execution-building" ] && \ + [ "$(cat "$tmp/foreign-link-target/sentinel")" = preserve ] || fail foreign-staging-link-preserved +pass 'transactional bundle recovery preserves foreign staging data and links' run_replay changed "$base_input" "$expected_changed" >"$tmp/changed.out" jq -e '.state.phase=="review-wait" and .authority=="none" and .offline_simulation==true' "$tmp/changed.out" >/dev/null || @@ -605,6 +679,25 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/atomic-kill-resume.out" jq -e '.state.phase=="completed-offline"' "$tmp/atomic-kill-resume.out" >/dev/null || fail atomic-kill-resume pass 'candidate ref guard blocks publisher-time moves and releases after SIGKILL' + +mkdir -m 700 "$tmp/git-env-state" "$tmp/git-env-hooks" "$tmp/git-env-work-tree" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/git-env-state/" +printf '%s\n' '#!/bin/sh' "printf hook >'$tmp/git-env-hook-ran'" >"$tmp/git-env-hooks/reference-transaction" +/bin/chmod 0555 "$tmp/git-env-hooks/reference-transaction" +printf '%s\n' '[core]' "hooksPath = $tmp/git-env-hooks" >"$tmp/git-env-global" +git_clean init -q --bare "$tmp/git-env-other.git" +/usr/bin/env GIT_NAMESPACE=poison GIT_COMMON_DIR="$tmp/git-env-other.git" GIT_DIR="$tmp/git-env-other.git" \ + GIT_WORK_TREE="$tmp/git-env-work-tree" GIT_CONFIG_NOSYSTEM=0 GIT_CONFIG_GLOBAL="$tmp/git-env-global" \ + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0="$tmp/git-env-hooks" \ + python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/git-env-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/git-env.out" +jq -e '.state.phase=="completed-offline"' "$tmp/git-env.out" >/dev/null || fail git-env-completion +[ ! -e "$tmp/git-env-hook-ran" ] || fail git-env-hook +[ "$(git_clean --git-dir="$tmp/changed-candidate/repository.git" rev-parse refs/heads/candidate)" = "$candidate_commit" ] || + fail git-env-candidate +pass 'all candidate Git operations ignore ambient repository, namespace, config, work-tree, and hook settings' printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/numeric-publisher.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ From 88ef9dece0fa2418fad9a04498a36ab594e1682e Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 13:05:39 -0400 Subject: [PATCH 13/16] Require execute permission on dependencies before snapshotting them Review P2: the execution snapshot copies caller-supplied dependencies with execute permission, so bytes that merely look like a native binary could gain execution through the copy. The caller must now hold an executable file for the jq binary and the closure helper; a non-executable dependency is rejected before any state or snapshot exists. 34/34 focused checks pass. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/replay.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 22f246c..d76ec25 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -180,11 +180,18 @@ def execution_source_bytes(repository, arguments): read_bytes(trusted_file(repository / relative), MAX_INPUT_BYTES) for relative in package_paths(generation) } + dependency_sources = { + ".dependencies/object-closure": trusted_file(arguments.closure_helper), + ".dependencies/jq": trusted_file(arguments.jq_bin), + } + for source in dependency_sources.values(): + # The snapshot copies dependencies with execute permission, so the caller + # must already hold an executable file; bytes alone never confer that. + if not os.access(source, os.X_OK) or not (os.stat(source).st_mode & 0o111): + raise ReplayError("dependency is not executable") dependencies = { - ".dependencies/object-closure": read_bytes( - trusted_file(arguments.closure_helper), MAX_INPUT_BYTES - ), - ".dependencies/jq": read_bytes(trusted_file(arguments.jq_bin), MAX_INPUT_BYTES), + relative: read_bytes(source, MAX_INPUT_BYTES) + for relative, source in dependency_sources.items() } if any(not data.startswith(NATIVE_EXECUTABLE_MAGICS) for data in dependencies.values()): raise ReplayError("dependency is not a native executable") From c15a42f18dfd5deceb5dc18e369411872dbe009e Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 13:13:41 -0400 Subject: [PATCH 14/16] Add the regression for non-executable dependencies The previous commit's message counted this check before it landed; the test insertion had failed silently while the fix itself went in. A read-only copy of the jq binary is now rejected before any state or snapshot exists. 34/34 focused checks pass. Co-Authored-By: Claude Fable 5.1 --- scripts/test/delivery-replay.test.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 4ae7db5..e41d446 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -124,6 +124,20 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target \ jq -e '.state.phase=="review-wait"' "$tmp/launcher-retry.out" >/dev/null || fail dependency-launcher-retry pass 'a corrected native dependency can reuse state after launcher rejection' +/bin/cp "$jq_bin" "$tmp/jq-noexec" +/bin/chmod 0444 "$tmp/jq-noexec" +/bin/mkdir -m 700 "$tmp/noexec-state" "$tmp/noexec-candidate" "$tmp/noexec-scratch" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/noexec-candidate" --scratch-root "$tmp/noexec-scratch" \ + --state-dir "$tmp/noexec-state" --closure-helper "$runtime/object-closure" --jq-bin "$tmp/jq-noexec" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/noexec.out" 2>&1; then + fail dependency-noexec +fi +grep -Fq 'delivery replay: dependency is not executable' "$tmp/noexec.out" || fail dependency-noexec-error +[ ! -e "$tmp/noexec-state/run.json" ] && [ ! -e "$tmp/noexec-state/execution" ] || fail dependency-noexec-state +pass 'a non-executable native dependency is rejected before any snapshot grants it execute permission' + + snapshot_interrupt_wrapper="$tmp/snapshot-interrupt.py" printf '%s\n' \ 'import importlib.util, pathlib, sys' \ From 3dd8bacbd3470e0814d6d20a3e614764f80d7beb Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 13:18:20 -0400 Subject: [PATCH 15/16] Treat deep JSON as bad input and bound reconcile staging Two review findings: - json.loads raises RecursionError on deeply nested input within the byte limit; it is now wrapped as the same "input is not JSON" replay error instead of escaping as a traceback. - Reconcile staging uses one fixed pair of directories under the state directory, cleared on entry and on exit, so a SIGKILL mid-reconcile cannot accumulate materialized repositories across retries. Two focused checks cover them; 36/36 pass. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/replay.py | 17 ++++++++++++++--- scripts/test/delivery-replay.test.sh | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index d76ec25..86c0c3b 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -98,7 +98,9 @@ def read_bytes(path, limit): def parse_json(data): try: return json.loads(data) - except (ValueError, UnicodeDecodeError) as error: + except (ValueError, UnicodeDecodeError, RecursionError) as error: + # Deeply nested input within the byte limit raises RecursionError; it is + # still just input this program cannot accept, never a crash. raise ReplayError("input is not JSON") from error @@ -460,8 +462,17 @@ def reconcile_materialization(arguments, execution, input_path, identity, state_ existing = candidate_identity(arguments.candidate_root, identity["source_commit_id"]) if existing is None: return None - recovery_candidate = Path(tempfile.mkdtemp(prefix="reconcile-candidate-", dir=state_dir)) - recovery_scratch = Path(tempfile.mkdtemp(prefix="reconcile-scratch-", dir=state_dir)) + # One fixed pair of staging directories, cleared on entry and on exit, so a + # crash mid-reconcile can never accumulate materialized repositories. + recovery_candidate = state_dir / "reconcile-candidate" + recovery_scratch = state_dir / "reconcile-scratch" + for stale in (recovery_candidate, recovery_scratch): + if stale.is_symlink() or stale.exists(): + if stale.is_symlink() or not stale.is_dir(): + raise ReplayError("reconcile staging is unavailable") + shutil.rmtree(stale) + os.mkdir(recovery_candidate, 0o700) + os.mkdir(recovery_scratch, 0o700) try: recomputed = run_materializer(arguments, execution, input_path, identity, recovery_candidate, recovery_scratch) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index e41d446..843034d 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -137,6 +137,19 @@ grep -Fq 'delivery replay: dependency is not executable' "$tmp/noexec.out" || fa [ ! -e "$tmp/noexec-state/run.json" ] && [ ! -e "$tmp/noexec-state/execution" ] || fail dependency-noexec-state pass 'a non-executable native dependency is rejected before any snapshot grants it execute permission' +python3 -c 'import sys; sys.stdout.write("[" * 100000 + "]" * 100000)' >"$tmp/deep.json" +/bin/mkdir -m 700 "$tmp/deep-state" "$tmp/deep-candidate" "$tmp/deep-scratch" +if python3 "$replay" --input "$tmp/deep.json" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/deep-candidate" --scratch-root "$tmp/deep-scratch" \ + --state-dir "$tmp/deep-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/deep.out" 2>&1; then + fail deep-json-accepted +fi +grep -Fq 'delivery replay: input is not JSON' "$tmp/deep.out" || fail deep-json-error +! grep -Fq 'Traceback' "$tmp/deep.out" || fail deep-json-traceback +pass 'deeply nested JSON is rejected as input, never as a crash' + + snapshot_interrupt_wrapper="$tmp/snapshot-interrupt.py" printf '%s\n' \ @@ -313,6 +326,11 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- jq -e '.state.phase=="review-wait"' "$tmp/repeated-kill-resume.out" >/dev/null || fail repeated-kill-resume pass 'repeated SIGKILL recovery reuses one bounded execution bundle' +[ "$(find "$tmp/repeated-kill-state" -maxdepth 1 -name 'reconcile-*' | wc -l | tr -d ' ')" = 0 ] || + fail repeated-kill-reconcile-leftovers +pass 'crash recovery leaves no reconcile staging directories behind' + + mkdir -m 700 "$tmp/reconcile-bad-state" "$tmp/reconcile-bad-candidate" "$tmp/reconcile-bad-scratch" if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/reconcile-bad-candidate" --scratch-root "$tmp/reconcile-bad-scratch" --state-dir "$tmp/reconcile-bad-state" \ From 72bc74c4f0287feb71727d6148c9223effd79d09 Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 6 Sep 2026 00:48:35 -0400 Subject: [PATCH 16/16] Refuse non-regular replay inputs before the first read The replay opened its input, observation, and run.json paths and read them before checking what they were, so a FIFO with no writer or a blocking device could stall it forever. It now opens without blocking, checks for a regular file on the open descriptor, and fails closed with a plain error otherwise. The test adds a FIFO input case under a watchdog. Proof: delivery-replay 36/36, shellcheck clean. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/__pycache__/replay.cpython-314.pyc | Bin 0 -> 62704 bytes delivery/v1/replay.py | 11 ++++++++++- scripts/test/delivery-replay.test.sh | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 delivery/v1/__pycache__/replay.cpython-314.pyc diff --git a/delivery/v1/__pycache__/replay.cpython-314.pyc b/delivery/v1/__pycache__/replay.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..615ba48f5a797dca5e01b2220f38e41c63276aa9 GIT binary patch literal 62704 zcmeFa33OZMeJ^;ig9J!$-xqL|Kyi^MN}{NRqC`sKPG3@_B~fB25?myTgfBqLV(gf) zn}%}IN>2Qq>DE)zsZ!G&wU!f4t+eh$PMth0^&B_P z@tlV{%ki34jYorDZL9XU&Z8?7!}K0K`)2SM*ssxJWWOel3BS5l^YJ)OTsUsLC!T$? zcr5JK>anul1Wy9{P4pzPUz^9qev>>&>^Iqyd?uxcs|#1FXNy!(<(JpjX~Z{A+L^Sm zH#shkJCn{E>L2I0(XY~5E_cS%Q>mnoo{G3LnfS6@dRKnWWaFEOM|R&?Z;#gblMI3LIzD#J&6j97GnS7j-+Ea{oRWI`Tauj-|0`ZE~;ZluJnxlw2 zt5Ndg_j$JQ2RxPhK~L40_@2W`ouwz#!79b~R6j6If*hxYPvmWU5}(Yc@Tq(npU!9C zHxqwZe0Gh=vz>4B?BEZ1YWXJ5&U6m>=VWqx?wLBdRlE4SOwO~L&xgNlW^EG&HJ)`5dXDTDw#_vFAJ6{XGgWn0ildpr{#qWaO&F_Za!|#Fr z7+(*c=l87=NbR%GGMm*m`=T(NlhT%bxJ; zj-CZ)$|tnE##7!@KB<}H#8Mnb=Gyt$Y5UBy(>^&pJmQ&r(phPr85^6LM98Rf3h%;Y zyWn(94PUljn)FP}%zEt3r<^0Rp2_KP`^fP0=;Y|I$7vs#8Ff~7IW(cfk(p_a^C{28 z;pyRVr+^TB&#-4Als3e-^mH_x9BSrU`&;;-!zcS%dPC-+p^L+l(?dg{`0k;u?xWou z9o+*Vt#EFxG+gMYKy^TwBTnb2+dk=W+uf6MPW#m4#Ys;{Pj#7VjH+paeQIWS)Hy0u zJeA7s5blYdAho2mtoEkni=hMD|giH!DlqHsqgs52{p5Bq2}D=bOU)IW5g?keJy-vYgbdpP!HeT+0z%&Y#&i3 z2gikXTJBy3Lg%?NoJWH*LVLE^qr=?L^V$;5p$l1fHsCFSFeALlbvX<)PSe8|oit7u zv6<1?DSEaJ4LvbCJQe<88yXs$6x<$~JJU1xoIW(<_6&O_M}~$y9%1s_tjFmd8WOTl z^gXInC|w%2nkRSEh~r)Dx-4Yk8}pj+;5NYf1MVk!vrr&~k4Vix07n}<{oQMWH_r{i z7hO+yAxQ7^Tg3Hq(t_-voTHp{P?d0RYP!%9rio$q#MI=ukkLIcT)S&`C~m^})ac~6 z)9t~eF+4duHS2U!ZOVm0_#vzKZRp%(v{j&mejI=99C-8Gx+Uf6!50oLmia99prvBP zQsJ{ydi9m8VG(UIB0g&qt?)-_6OY54*X*G*aGSTxq3e578;Wz!2%ez}&dcslyxZv- z7KS}Df*T(+l_5<7nyx?Zo|z6AMrSX&+##cLnpU__hTPTdjl&KwGd(#nJSEiPQwM#f zVWP&`t~R{TaLu{oxfSoX90^+5RxE8kONUqAAvRb&JaU6SLg>R9tQulH%8GL8lYUS> z^_4U{pdaZBQpa^^bJdf_9F;9zi?{);bjG@Lh%G@|Tu)N0*h-l_DY3ry>Ex-Hw(-04 zO+GDB=?oZ5kYe(>KCY7bE$pQD6PjwyW$4qK)u=%xm$A<%?dK9iJIJhM8>`V2kUQAzOHXOe1B$n_ZqbEOzcIF4l5xs2_YFucAeUk;KV`V3M| zF0-m1X>LWiB2MYYLQYkt3{&-K7_?=L>OZ-J2~D4wPUb-dS5x*lanZUw&ZSrNwhN1FS(XI53RF&8XE-~&$;Z$X%~)JoKn*>9-LvrqvA1D=I)v+jQBQ; zuj8{*!-AcTuLgUWn=P%mJTX@6L(`EI($2U;W*jQb-CZ3g#iO_Da7Xu%_K<1Zi7neB zg!D5m=XA)_-OEk^wjGD`w4H}cy+f_Ne9N(rVN85?d+11ThV)d`ka5_Bl%pZt6w>R@ z&)^g^j7-hAosPJW-bJ8*K|H$+V^hQ9*nLKw?h#?qh3zL~aMF~u&ce<-&)qk23AU^4SK41Key!@2 zs(^LNjV+aAoTU&*Tv+V-y6_O7SodkY$Gmj()s`cjU0t;g1qQx}YDY1xZS z|E_f*?x%^F>uDJ;AN%sL#Wr7B?Lx~>Q?u5R($}mhS39qCE|&SN_BCtztEr18{v;=0 zEqS4J&6aZYsVh${8hy6H1^v&>@mI|+n6Ksd&3Wsl_=URXFJIf`H)X9SW_xq$eTjR$ z=Dlkc+kE>+_sckI>U~Z_ZJ>|4Zs`L#`+bQAZtwD%kN@Vbj=ucnegPL}5pxWf^FDTO zM-Tm7M)N-I{j|L;MclU&OUd8al#ZVd_E}mAbl)+iwq)zRlWm~qqO6v^x`;!Hj!$+- z{XQP%fu*VDyn1iPS90p3m#sE=?;ezQu|av)-KoPG#IA`os2Q|~Ibg4l5AQw=L9;GR zqefYE4((iW#PVqC?e6Mw7=;?7$E*WPi0Rf(XEvMEp_CIix-fH_o#cyQi9CQ?!96)M ztxSA}R%k~S!ZCPmYFfEHJnL|7x}8IGQVKl6FHnSo@aDO@T7#j1=Fi*rDt)A8I8KXvNq`c z#?Tnq_tfvK@B7&lA0!ZjIR(6NY5FxiVSqsTO1~0_{$CLoZ!;{(&p;}F-R&P9Bd z@{;0?0zN=oZI3*TN*ZN%?ZHkQ?G${e{t_@_aj&*Xv6OM2&>;61r|sFsO1Td>8N=F; zj|wM7i*TYwJE=v0gSB%tw7&rlniQNP*q#{TKwI)K5TB_jr%*L>X&P9D{C%$A0SR0{ zVvv&~UZ4RH8sLTEC*4Et%NGGHUkDlTj9dU^dWL2vM+K@y$cU|&UKluhfI>`y(>*iw zr1MRkaD?(HCyz37Q?}R;#aWpgE{?JkD=wVEcMnB$!vk<;O}du$#kTpDHA~XfhARzU z+&|yEX0irNnJcEupecLRl)aSxM&9drZxnpB;AY*s``+I7LGgDT-*)`{Hvi5}w#l^n z5<9%+jy03z3r~OU>1)oQDc5hxWuF>+iTk|feU$fsD+j*V=+)9I*0Q7hopz6ViEOv(=aRS+ps7`J zv)ZG19PRM(+U=KD=w^FYm$n^HRUB8qxwJ?v?_&L032IE@jso+#gw0y08z_l{g?Wtz zH8_gla2W<#h|V}(LZd#=(j%Wla^1PTcpPW2_dwqRzhfrnaqZhUZjfLT=6Ja-6B?>% zJMdHbB`VM?Xg;&3`6RIIUr^ft*hOq9=VoT7M~8(;r`tXQ^2gM$%V7}b1dVVg&N)4M z(Fv3d69PvN$Qo`o{tjcvKp>Wz?GGVyc)u78878Mkolk}IQ_e9@$RJFPPk7wa)(Qqg ziQ9&Gbka@W8c=n7`2+m9XW-3qKi;w1XWR33`f}P!`fGh(i4P_fuOt)KY|~&w~A+d7`3;Aqq4*P zBBJ~e{@fxRWnGW`^rg#7>AuV@%VmMgnh*8cHzK*T1Xvp;;6Dt>1)&eLiXJ`EL^CHk8F=WJ3i6L!hK)aM0$Vjmg z4>i#DBrEUIlayXsv=|>9{gcr1hjGgphW#VRa# zqf2m(O+FRU5xAwjbgm27u-W6h=n_xYix+SvUvwcF+f~DW*w`pTh7Fg5N$j}5TW8(C zV#nNh;)pwxErCxycZ5jSL`^QlR<2jsNbX@dKJ zn$TVoGKti+QQ;`P#Ze7rgkj-w$T$H4J5K#L8IohWwut+(U0ax<%*p#Q{@gltz^><& z2J67fxQATdE5tZ1pE>T|P=UX~&h0 z#Z14oDE$1Tu61kT)#EQ5zt;cqnHSG|Wyo(W`<01Hblf)+9vS~a{I$J)bBToEHow;X zO8bqJw+h}Y@TJu+!~vy_vr9Rg^jiym9?tD#0MC|$t9!2Oxq9%*!9_>VQo3p>rKqhh zv|gM1&}#psjzd|QC=2!b&HWTY->D*$^f#NO4Os1WDecE@0+;{1uw^IrzM;w3VlaH$ z(v*O=zt7HS$>RRLCIipkF<4sCu!;;X@|+5uq(g!^a1ZI2f&)Cs^~i9A7)HJVVE_h_ zX;=F+kLlaEQNUGs;3@ICOyF)wGS+w0s$fUsXcM3oZ2f>r^qt8JIx%!6;OL1;CDlZ3 z8*)KgV%GJ6H8K8HHHk{~Novy10gE$D0@mZ^@&H9mj)73^0cLL}7>Qu+$!SpVCr3k? znh44UjrUIc-6M)dF?w;H>*J0VbKDcjyq4g>eHy7J#&IlBKLg?qj}|`54o6uy^eJxM zP{_5HDlp(K-K55$4{==wsW{>D@IspEkG0j+AM2~fCp`c}{(yzq!9z2HqUI+48wm8; z>g_~G6Fd>5h`9aZ&S~dUF5zG(&pkZm9HKT>@1L3hT*AXiZ?*ZVkNGXfe0?W>?WPJoS5c|yI~vhVG?iI5UB`pm zP1wU+U@G-%qB?4q*ij0RC~Bf=9&vhT!xL4O=; z^3BFb8LhuSTsNJzY_tj*zgc>-^i}s#-Je`uuKm-`{%vV^Y;yB4Hmu~~>C3_wDN%HZ z!c~N_?1*3>4nF_AmfOt>b=T^y>q8h_C8 zYd67G&m}b_X})5C_gXT%Z{))R?g5g<F<=>bbh1amd9UvgrT7l zJ=B^^gx*D+&zd*i@>5g%^X?Tk#eMOozTp0X`#I$`XgalOI;Bn=j*d?8#q&~Z@hxcL zS4|?qE0ttPnh*1_)a#y>1Uo6H>cbk6!0G3RTR)>wCjKCe4RcrmTI|(D;?k(bqS*;4 z0mvzs2We(9r`o%bGcaG2L6dryu~LeH-x8v)Zkk$J8j-0$X zr27H;At6ZnHS#*aDd5udYOT~t4O*!SuMB|45~jvSa@}%yplFBh zV3)W&o=?lvfT5Yt({!Z0=~&Cq(bf)#9vD2cE`U}^$6C5t_@=(r?k@2IfW&yECxj&` zh%Kf=BAT2KQ8nSu@%=4|Ljpk`&2wuBMN1V+d0tEPeDk^`>$P34?(*Ba0|&c&_U_kq z1uWfOefOdgT|K%0?w zl*6g7=~u_0@}r!Dm=Vf2hS87UN2=%@K=)$MzZ8k2&5%KtVG#ST=oB;w6eHuTwuUH1 z+c4{lC&&7=OhODjlm`xAXF<%qF+N(~oYS^fiF4Lvav6D@(z`gnbX}l!}Wpq0mCnZ7jVpSO5i}!J9;4%gxiCmkA?~LTyZWuJK{gi5?Q6#pRs3Gk*JjbsjX`F z#68Sk?rcvt-`8}wqXlwCe9MrsvmMy7b-YR~&g9rq`f>F|&<}-40Kjt)unO;Lc4g~} zB1(UpS3T+^&MuHu&?ReT&YcHyp^8|$vx4*Eq{y%5pP0*&qsxpFQF4s9*|DLpJ~To; zT`;$wNR9NWcjTxC4?d9)dXtkPc;aVbkw#8ESdF<;V%K z*E*YywI1mWLvD=ar-n{Mz|9esew&#da?i{PBTkWa7~_cg5z2IuJPIh8JOivuQp$X@d%L&fY`6q^$ zz?vx16efHOkBm6o?vVcRq3(9ZXc8D7$^bIRMYjmdSwdjFyJp;z#2^dB4GZI-OoJXB zN}wq^Bsiy7A`s|5+T;idvDc! zkn-KU_w(-Qv+uF5e&Uv+CIXmdsyW-gE%WJr6 z(B&At`i#3~F10L}vTY@0+lMJt3r291lTxl;e&Od9a`~H#bW>UAz#X_Rr9VN+ffb9ko-#yv3_Od%oYN-Pv0I4wmiPl^0>d{ zlrR3Y1V~2(2Ti9}O{ZChHu(|{d(DT}EGYp?kyl^z>wk8j3x0hs54pOrX>#Az?Krka z|DBq=WBZL8_6-Ih{a=VRPvr9OX^U6cNjto}4`ZhUhmY%_lu<4^?l^d_EG}T~XtyCQ zI}Lg4I0{{)pF2A*om#*sWvHts25iOPFO3{q+pl#p+5-69^69G7L4l<}#3ee1w#!XN zz-8=kBNwV9!OALmMN~bxqhV{r5imIIFbPj14P!A1grA7J3c=Q)xG{*IimR2D4&di_ z;&uxRB4-Gka2w%~e8yD95GSe6aR}`Ud1fv+r-kn$nxCRN;bHYz(u0<~6-!>wQm|?% zSgHsX?^!9{6D;1lTDWfVPwxTY{>qO@KGllJTR2%G@Y zh4Si(jkQSYT!J-Ras6@fUeZq`4qVxb9vW|9rJEnAw`#(1K@*O4Qh9P2B4bCviOkBL z(r@BTm8uL>vNJJTKbR&(jxF=qFYDrfqG@W;j4`5Jzu}QaHG0m;@|P2uK_X!>*CXR0 zQV6l;&f2Bkjm7MedZdR$tfQRt&1F)RVULQ0S70~>Rg4%oj_Paa#`J3VToqHsJd_j> zF&$F4l7^3qEq~0%%h+|i61xYuOteE=;(3d7Qp8~BTKX+LbQnZA-r65ATO#LSI;L%tw!uU(#PmE z9la`4X_T7ROR1G#Rzk$?z$Yqc`;D=5zNCr*dw(X3zy9MuN5r|2*q_+5Unxa;>cQHI zcF?GTPL;&oSGRHCe^b@O+S3@F65OCu3?79f6taL=9Vj5+tt-GU(9H?x zwoQeM!bOkZbc%?uX;uIue&%9G2lXFe2jxPmf@J6icqT({>K;16w>0&IjNPCp9O)2f zcM@jFdy>3M0SLbdNIJ=vi82QUHcrC( zNXFhd1jU`m8!>xwBGlB%nw+{H_;efE{l1`BIe3u}OS1}*t3mi(ZlaMe<{bm@)H zzW&+ZmOZOm_T1X_HwRb&oxYSVueEC}IcpJ>ELkwFr(|5a^zySWKI_Ze>PvAfn0{(W zf$Rh%D3e8^4?Qeq!)Y&+4Hb!e*Xabo;C&OS8+fUwg)Ht@}AC z5ri{eHz!_Acp>51xZhj=aop#hiNcjwMOu9+ZC-2JdQ!^E@n4Q#to=%&D39k$Dh(!W zTS?mHOR8EhNXiVSAd`{%X6`$}yK`^P1sqKu9R6g>-=X!qWb$&VT<`cgBz5k-q zR5YyZJ9JN{$vCW8i2wNp36YjHk+bH_s=u`>kag%byyJdrYdH8zosy`>iI)do9Q?{z zzqM2n7BQHrT7frP@PbiaPfT26T| zr*b8y^240!D~Su{wT%2g#?}SPnmILS&RH?%1kLspvwb=3mfmZ&`^^W|&1u(8`^{U{ zvh2Yu$4ZuCxh$BqeKl+Q&EvtkV=HyXyvH98)}36fJ9%Z+J2bkWUk@u+#M+7>SP_4(AfrZ1Zo`a9NLK z=|%`ml^g-_^nTBXJ#APUsbm3Sw+_t1-E_#~0MdeW$T8NToP&bgZH7Ln7>oenCE!5> z2g1T3ni7$r2lY@Kk-vkIqyw1_O@%F`5J+}vq5i?pIZ}=$ky-f|j1R5n zGHH(XLHSn)Wd}w>70)P=>ky@lb;?mBR-R^FdHI;gD9wC8wT1tTSV>e-IuAc0RT-lp z{in8+FS?-=lN$f?;`&=TzT)m+@rjk<6aM0Xdj^DFb$`i?_Qb`%ApR_D7Wp2hONwW) zKLSNIA?>BpUP}>aNTr8tX`~O8{;|6pg+CwPw2k|tZSdaH!uysE-ut>OOxfx2#FpeA;{igvXou(&uEa);mAt4#T%MLSb4nU%Dk)1qB@-r7s?>Tg3f*F3 zE}#c$<|;pl5Oj4wWGVKxurQ$OWz=hM;}52#%&ORDQXmB=`(VhIOaA{)qY4) zwHN+(M9QJD#x{lS0@dB0;kdCYXxXu9+3`-Ve{Y9xSEs*c$iHid5rdEW5?j6I)^$s! zKWp2A)T3$9voW}ZoYiB{4_c@i_mX<`;kr=DnDrQudz53eS*mmBv=JecHoO21wggz<(V9Y7B8t}#qVcvKpILHQ2I%H~*GCv0lv3h1R%8t}~ z;I~$X`K?Bmf!6`w*7qC1(Ke~dNtf#->i|pXfsY$#we_21&ZYs}Orx-jKo}T+_ZRR& z2@3BK69@vfkgRnJCHUm9h^(#-iu*QsUnlPyn`c^uZm zRsvAi8dOpFNLB>2b#;gVAzF0^uT#IxSbMF!ULC?4)Nd}ee1H{T54iGEW~4ExV<}*b zG!HRfzUVsi4O!uXia#t!4kfa%EdS5#<^c`fqg2M~|~|l{y-GXkkS;(BH!C z*`x(nu;_!$TVUi9)MF*(`oQ)iN--ka6SIysDefk9gme<5hK#&TT?@XP==fosQDqEe zS`6$2XnV4vx&*I~KzaCFiHq*>4Rb}N-=U4*dWcR@YH-E#DSa9~wa>t(q2>|&$EWw1 z_>8{TV}28*RQ zXE}Aa`k608D*IrW6mw7-Zf?*ah4tz9qFCqDV1*Pz>PNYj#ShJ~_Tgzr2|*&vE_Q-lf={|WwyJsmb30;Z5}r+RG8N%=P9-j;E|)U_>vDg?ySpkFO2 zen7%aG|i-n6tap#8#e8f7;gPg_&q5+k>qxS5+kP8Xed4W9$DFtSt>H5MdM{NYSqv? zbB;2hJxrx4v6Sj3cL+x?&>@>tVX63pFe(+UO{&rZ5W)}x8NeD!i})gmQl&={V}>lM;TAri ze9e^6F+356L7Y9px7n+-jKn>H1O`9@m+%97STH}@&!~L|roZY}E%j@r1aD$Nz*Oknve$3g%Th>2By3qh+twA^*7@V>Pz~Jw!v00u za#qk%yK1Rr@fv-JhrH%P>xtWzTYZVU=a2o&oUs_UX3kpN2En++qf7e&c{TJ}y=E?4 z%6KE^^&D^IeqYgnH8W`PN#2w?Q08B1U7QRSK%-@+FAeYci%;D(>C@xy#&dC5uNL~U zE7r{Uiv=cMuUoSgv%O_U0@+7=*5(gpfV9qt*B&L6*D$>naj ze)(pWZ%czOXW#tsg_A4#%r#5ueD_E9yER-w`b!tDpY&R4h;(&EH15;mOMJ{bFz7W8 zGQ&P+G#@klpy!(s+brDoEg5aKy6+cR+p2WmuQK3y#E8i%Il_1tw)PINHAq3}$4VKj zj!7yyNX22UMmq_r*^G$Xsp-=L(=)`#QC)*@){mH@0lEM(9Nw?eag2g7Wvm6(ZNhEg zHL8>{^#$#tK8SKkOodv4m@-VSM3-{ktS6*6O0Ei}MinOWj#XirNQtu;Icb0?$7pRe z@bJQc7U^a3_ z@mBRWSNuaulk4V!>`f$Y=CVGdti*`268aMq90S@5GL9s*SFzwE^#|+e%hb~-I})Rf z$h%+1C#lO~>CUipaeT7+3!f74_DS;uO2O)q8j(vH%1@67$v{YEL`W7wvLixr5R$77 zsnM&X+?3goC-nyG?B=U_aMO9f7xX9ah5c4OOI-qQM@UXYND)Gc)gd(ohS>qR5uJjc z4$7FLYSvUs@swX?*B4Y0SU5`@RN7G@wWBW)cpttsCaUnh(yow24ek**ClCxL(5@iR z<|Yt$BGBe_^i6$8=bV0$%u0>Ia4-9H3NW)bGT|J#FlSeC5+Qon@@0}9fOw%A2U7=T z*YIU#Hcb=?AZwqZyBGxv_V6nM9N(BMG!#*$yiZmF!_^4oC_$n{2vzi?=#aVM*!6|{%*ElUu^Hi_bJUT=%=ke6 zLh8yPQh^j6$9SRlMgT#NOo%}YVE!r%CPBC|@&VOwW-Bb-J$nu!IZQ!A2T&abh+BJF zNE^d5D~Q%w4^iT+)VXKriBZalY9Vxy$6)Cl^prvQ$G1YY=PA+X9T`px*~omJ$~2zC zrUIkmMPQdG>OwJ3L zm$MNPNYjjfb9uZ(vwNV}F`!!kyX_*&DT&Y&b;k?Gk;6Ft+|vMNAZ^mrz1lSJQXfoB zJFyFOypmp`YV=vV$#s#ygSF);osWrjWJu9hu{Hc4r z10(YtKQ~(zT3=0A8V+P_4OqAO%#P&)@0Hv-|F?iSlY{2+6?6Fw-Hqm(*})yHD?3_! zm2JUFex;K4RrdL|o>)ifE}uE~)!wB%Uw$>cwtBO62duk&<~{GF-a6@RJsmi3+P8P` zZk)kpoIiHg%BAHjHZC8(IpR%em~UTC$h>;?g|myt-aGZJp>GbYmsGvc_AWI zZuoM;;@E2!U%BW@t%QI@CU^mX%x%j}fz0aVCj*(g7p#zPdAZ`t6^nzulnNj#X}OE- zfwZj)acda`i*tdDisc=FjH+cIFnbm(KeJ}Ox_{Xk+`4aN>ptJs1HRmYe(S+CYwER5 zB5l+Mtd%#--o34X?QMQ*+nP1)+S!1$bg3WV%Vz@CeYcJWtSxu4y!^R9>xkbvLMaiH zzXl#fmbcdPT_ATCIYhkfbjFz9XI-dRZS~ZO;saWNGuDFR}6V39q@0 z!Q$f1D+8TiamE5{dt@f0lu7}XVzCM&a<mmWsehpJ{W~0WNqQCNH zghbM09wV7CEsv2r!ymD4P1Lsz>&E~mB$Y=p)dK3O zWcI%XuOG)(RpF-!Vwnf#V8i&yDnJTu=TV^?hmx?9Jen=SE_ie4fSuem6R=(krCtdp zIP1`UoEGcDTyCt;v!5Y&`{0FabeTX)SAQ#HRy$j|`h@)y zYHDuj?>f=ZAwmkoL=PFjQJ8cI2PkDWc|@`k8sUMVPA5Is52Es`2tA2%K({CZ{^(EA+8500$+<5Ve7Rt$;AX?E^S1`QJKFBFdruB~?dR4@ zcfQy5t?qAj`}@uY`^HxK#=O&R9I7d^ng!#}k}_UScrhWERJxK>>PspQCRMH^Rr->u zVdKz}ENZYVY5bO=8>w&QznOn?Ht0C8>NpTe&0HfX&MOJmoFAI=ewo0fcW9tgX--@i z{`}=@oqki%T2k54lgquPx&7pw;qP2tOR8MH2(hH319v%1+<~8m&-=5>bGIsP z<$0@)-m!R3OnLJzuG?$gi~pAGn>K&XsbJ5!m7a6n^D_hnJV9VULh98sSIz_zidGVe zdEFs;{xgLP|L?((-2IG=k8BAr`qP7W1CR4h@4T@wK|m%nwWb2f5DJ zId5{(LCJw2edZJa1f1TZIK7QHi)3<)6iYcJrJYeUe~-8_Bl^K{MKfZBT<2lbHD8G* zJxQmm6MYO~kbDv3AE2gzjAP$?T}HXF5Wanuki2J{FSQ_c8R96%{uWEkQ0c>5>g9*_;w1M7qlVS=yBjsd zI;g0S#f(&C3T>EpbJ?)~tZJCiyCAgVyU;StLhTJux$L!QBHY$blN^&ZfEe zSr^@jRfU}9OjT7NzD-U;+ItI95@dxp5!7-}R<$!OW|DAv21@%#A(|+O7ulbnd2Iv^ z?Q?PQuwqKor})YL6(DTaT!jotA%z5)Dvb5<*H7oU_7M{kfT`$r08rZPgP(C+aT(wQc|b-0(TvC=?I#cP2`NB#s;vJW5b# z$k0QERD~`)GuTrAlZz8hkoP2V5H+hV;R!aXho^CU*dT@WQGnhx11WSm*g|^BA3Ztc zbh;Q|%K+7o?XW-^PCb(n?|CmJKFm^F8pdT|iG58zqhddR;l;UsWn^f;z@vK;Fuo!d16*Dv0yc^VDZw8?hhKi z+xULtdinNXdHqUx{RcVUEquQa0HTf8fUcx3GL){FZ3LYL&ABV)+_l8Rw;R2Od%eA9 zz1E?ib!^2t=Ce-R*JeE-A{^Dn0d4$>e?($`Hdy~3x?ZDNx7uv4PerU~y)=f%& zm{KFDKc(iseDKQ$*Aj9U=aws$^St@HZzbN*d%H%xR_A(B;f?s<)`pd>4IeZFkDORJ za>9G&92qEyUobi%o2tLK^y zcG@-HhK@v=j{87+cng01!I6Tnf7rk4cs2Lk9b55yCnu%Vz}>0T(sOlkt5$cXk)FS& z(ct-eS{*$b_Ox-j@9k&LjT$_EpQGpR>ry-Ob>Gk2)t;dLfmMU&A0#BV7ny!gprvQK zhMtSI;rWLNDIIqG4+|;I5AAz8b4>r3Npb!$TSLz|I(p8hJV(N;OEiGv2#>)QiVN}J zM^fQpvBJTOijawugb_*TehfK?^fD&itYXsw>efdxZe6B<@ts=0mkDNa{H7ETmgN{kZ~*+huE?T z9NIn~hT~;wUo4s)(imY>4!b2e;5HA74P^r*HWa>9qoWEALc0&EtC860 zDrNkK#m9xS>iGQz(C6ZKvzl!P>@SXwiwKEFNW40vMyJ9OlwOvFOrb&&$Kc|J)CWSs z5%qakdm%y`5nqA|9ahGh>zYdiQ)ULn-=<)BPNG!JB`bkp?$sP%kZ?SESRa{$?o>B! z3jYfRpbwBj!sPfgSqTGE5|imR!;a( zk{lN93xfBCVPzCtIY1QPQQ{6$A|{adm`Do~NnwCG!(FCgWih6#a1j;w5Pxohl|T`L zn3yA2+^|yIaO=3YWgxKsai8tv{P8uDZPk&)w`jYn${VFMF$z8GJLg{(ck}uu#bG=~AmKL-XuGk8LwxU&A(Q^7*d2i;uRqz)D z>=1AEWwdycV1)_ym%V`d%g9oD9G6mmKb}-)q2ucftM7(2dc6txi?e=x8Qp}ne?7fy zY4TRvT6sMw8FxU*7?#T70E)bj;LX^I3(H7}muWwHGsQE}i%8=nRy1L4YgG2-!HZ5tX2uv`+aFPkYU$K^^q!(?7bOyh#xs zn)iLo%!~bfMM6sk_nnN|qh?)%a*KR6e0{^&AnjtGfWAeyY&^nFJX*_kV*MlS@*-z~ z>d9P>yjw^iBzrB@RX#_sZ-5R%Vgk%X7@F6BHU(Z18S{nkm5ir8Km|re93IB$0e7a~ z09uNy_Mu9fCY7Z8s=me$9^y?Z>Y5RTS02op>14fSj3rjI87>}+A48h}IjNsUL8(GN z6^Wtf$(W@`y{jldsDzi4K2{m~RJC0`%cL^MP!VoGQF=B0U#VA#LiONYt!6DRlWK*y zGWSag1AK!0h}@(0km^)Z`bBlBDe54bm*+AMJRdVJmC{o`F7$LHh!}mPd|1Uqj-!qy zG#w@|&@>U#H;)W^sz?zLIKNv&cIWU8L2gB`9QjKZbuwWTrMz=fGv}a;0e^0LbfTHr zDN)*nZmx`xZmu9IMOc3Q;-s6&y}$-20j^APM8)k<*fs2olbJu!NH^o{Fxe0?SVk9& zA%7srI$Qu1%B0K1a7%aV(N^dJ<7Uvq;Y-F2QHiub9C0EaNF~QYJSgD^MJS@~dk;?` zOIYJVH1Q?;Cr@h;nZ=d(_{ocn% zyg8%m1y%2ye|P5XnIE1HcKzN;*Y9~JE&{=}&S>VFe3SuDa$cuRF}?0bD*u%vFK zq|R5eJ6LjXrQ~3+no%4Ri@@9#^V z@7&1wF=1S3;@u-q31-RTp?<6ls~@KVuGS6|MQR51N2q#2nHUf`Gv2II8#IqCdJN5> z2-VO?Jt7It?^i?2@)@f(MXff;`2f5wf{(tTT%#I7k5n{$NZbhczfO;yg9B|FFnstL zI4zW+Eeh}(vMJI`A_&L?tW4lOOaRHX6rM%~ zCLnL_uOZny2ZOljLmC)#NHYFY+S!HXyRa#n;7uzFB$h9E0*RHY=E@(ZoxKY+X*UzbJF@#Z8Ff8RMsYrqYxT1x`oGR3)k_I;@H|a*fMAAxQb-Fnl)vUYNoaus;*hga%J0*3W~m=-X#pMkb%Tl z0y{vPBNPH9Fhgp4SZ`LTqbx}e#5i^P&bikl(*1>D~_XP+4x!?j^f z`^-7FQ^19?bd`}>xHM$Ou{tA=%3W!!xa_M+0glP+GFg?RjJZ!Z|f6YG;811R>S1*a?);+_ZkeXtcNk0oaiHMfg0@r_#ow!vlmsO_t}L zCVRpOnTvWqRDcrd=8rOST(F1()56L5i(`;8fo%qM(M(^^QoL#@UUI%M`TC@H`ypR> z)9uInhqv#PukEyVXvE((!nU8VcqL2w^5lz?Fgcd7?FMeEAuA{ErGKmN zn}v7M@5}{SPX+g$UfnC&y*}hiZ1S3$VCd@$mp^yen_T2G6)#PAYZ?RPhy12Pw_C_k zm$z%!*L==9HhmXItLBLIUr4mIS$m)3TJXaP59@cWSu++5i{}F{ezoge$J>ruExy{r zw~yaB@3oxt>Q8Pw$cYZYcC>!X;NiI8_(<~S#+^)Bfukqc?!(1GXB7)I;oS!#3c_OS zxH4+WYuQQ^ceU7Y<(ss4CjTGa)ndn$cLd(3TBkH$nB(PDjU6L)4)RK#iC1>_-uMUi zNsDImDYm!C$awfJ3Wh|2p&t)B6>(&zg5RT9rZ@^zTwOh8Y@ZeD#{rAy1Me4=G>`&Q}>Vx}p^!#s;Z>rSJP4Yd+^2IC?U2LK|pQhLz z>5YGc7NzqU(#b8=L3)>Gt4Ksr)>Nhxb3*fgS(7EjhyeSw4;~H7!rzbVx7b3?e72PH zgL?{jK?efloQL&Q4o>;f9>35@W$+5^3y_)^B-u=PCv% z>$lEdYJ=2MW5t&o875VJw9_DkSd#grfb8%Wfe_efn>+lKEEnz0cqV5N5}q&l!VA7XX>hfV6dO^Of8XYzdRH*GSX zUt%MriW?K+sg&~FWM$8=oPb-4F4oG1UBF89J|X2Qk69;cn<-`;s-)O_HQRAT*!N+l zy9yEllR>7-NVS9&ecNvV`_xEvVdGNBRCYESm+ew~+&-f8wfPf_OH82-{&zJl=0_Tr z*mc+;wT-X+EypExzQ4`5?3C)l#>LLY1wy|Ouv_etp-xKq;29pKLn*6smy`z7pWV2X z34e|z;jX1bx zI5C&Q(%8>K`DS`}DuVdKC?+HFbi$M-k|mnU-TO8?m31#{bL#7jd$m1828ePB-&m1~ zj|N{W)E8lwQpZ9*he~CxM?My%5XRl)dbO~xtl}qb%r$KFt6$_3ko9Mh(0x-YXwkf` zHx0~Gtw>0E1$ir0g+S5%5(N;A8uV%+g2%GxCDW-{Z^p3bwH0u>y9b#_7QHHrztn5e zk7^d0$&Pa7wb_?1zj%2m?JLjTz*WPQesg8eeAa6|`_AEaJHFm=yU@!Icpo1O@MnB= zXZ_pH`pswWX({=y?-9}E`I@Fe%{MCH35jS>-%(K?K8|Ke3BJc3#QzyzRD;O>s|;fC zhCz%W5Vm0e!;?+n`WXxs?Xzft|V&Fg>}^;SD%eigGk5CB}QJg z$&7SG3@IHuY+d5CRj#`PDz9FZFn5C{i*DlzQC&a}jRUss^iu5DT2XXG^TvF_EA3F^ zN)!GNjRD(-5p5$^m~=U2IZE4bIVKyay);t$@R_8W!Nlp)#F2U0+3-kRPxaZ#{FX9t zgl_O}4gSU8t(rT#d{vM69c+BEc8Ip`p0hWZHGftHZ>}bC1QfoO>f+J}Lb$;N0d}OBSP5VfuE+D2l<)k!YI7Det5#e#D5*L2zFkLneGHa`oyWh0Ex(h+&qu-+v=3 zS9l36RPV5?$q!hs?KBI3DM~x2u3IB={KWn9*x6h1moZwA%O;F=jR~pl)@``4aKo8x=ix52};-E^0_%!vUz=p&hCJV=0VR06T0O8O-67>wdwzRapj-rMBeffv%b zVaM{%>Gc+Q1P(E~l%nZM2If3R6|e}Op$agm4%)bcm+=}(pLPOT-5~zUCj6gL{MX3) zDtUC&i1VKWK4ptIha^Lkk?^f>1JR3VMFatP{2BsOo3c2={K#t=K2w%AZ>Qh1^XBZk zm*2h&8<2ZCy^js}_Y6o;3Vo)cr89n$XrytsFL968yoas?YQ55W?Yz%g1jeddmd{fB z#>k(IKw4J8zW!c5c(@lzr_d}ms>~zBr$8z@LJp}ao*y3U)J9Db_Np~ zeF=>+-*l_bTni$?hLAJ1G>W$mM+E(f)0i9onY^D<-4u2$c)rZw zB!w=v!7yhqW7|r`wq?jY?|26WHutPr_ORxtKOXgE?0g4|-o2}qy|V4f8>!1DZZ-s} z4hE|ZuT&lO9_jP;KklnK>2sV4T28N8ME-BssO4H>>eb0BlU~>zqHD0Igz;BBK})I6 zQhKB9<|Pu|y4~p8)9XL+c(C%MukvKL94|P%du}!cw!y$=v(M7ZQoelQ#S2SYd>Iut zn!$bcS$3?O;urQ@+Y_+l`b>G(b=Ny@Zn;$xsBQC=w)^ZIep5%Vr`Olhi#wbFsQoOf zaLEKCN5QO`m8=@N#o9o@h=TvC4~g|FPWPB5?w^?t;qK(z#XYa>du89!Q@&iB@xi3s zt4X_Y3D?3?0aMni9XD`KQqJPBrS3pp-Oce^mu_DQG(eT6@XB>S@4{w^wZ*r*_^lN!V z-m@+R5bUeD_b$*XM5*eoz`LKek$a z>_Q^I!ki3|Z7fZ=c`x42x zD0N>_v)|ObhK3QYTL4IH!MbM2q>d_Ej1Q!gEHwvG95AeUt!1$ykXE)d7D%fi+fL%@ z@owE4$Y_9seqKD*yfq#Y`ec>qm@nnH*9vt7r+C}%MXTJz5PQK()n7XxPFP2{~ z4cfM>+P2)7yH)ljKILoZ@$)BwHT|nK{p+^0tIu9}cJaxj^MS%0L0j#rtriwJ zt~Ok0xHjpt*q4s^E!*CyeRu!Y_usbpAM5j;I2}AOv~ps|`+F1K^HaVP7k!US2Y1e_ z?wnz2f=#}Z!(QuQR-qEVsbnoJ`{mXbTNf|*(kd3>s6vkGeL-8rs;%Okid!eYb>^FA z?lkxg^!pz_9jqN(tsRsqba^=^P*@kV?OL_%!VP!KMsB{(vc-v?p&8}C zw1dqjvXUSf6onQOS7U%oz7=LaDMvWG#VeXoJj~of+M48cLYdK2E@-xpTsac~ z5{^*T-F|$)ai@>_Qt8!-7b>oec?F?TAS?EaGxP+-B&v6#m$jhUtWh6ro?UF#Dy1A6sl2+a*sHPcdYHa)xtbPZ){9 z8H;jW$6ALQO9+~aa%Uq%fMKjm6K(1RA(6MO>zP=sN}&Ze55dB1jbeHVi-C!e;XFx=i^NCGFReg_9 zGnS)?_-RUxP=rj9>hu6LOM@y#niN<0Reg=2vJ-LZb+(jBNyD2TaO-tkU3g0YFGQ)K z3R{%&0Hu=OWRK>6>U!*4~Z}fkO6gFT7*-WZoykb`t z;mUp87M1D>*RxehtNbd~VWl!$)_M zy;N{m-VG%%u5K=7wUy>zzisn-v!%ix$nQ<)EM_zTLuUcT@Dt}NwMFg;C#&}Xsos1d z-MP&tE0P>!o`$#3KWXGY(Ta?<#*IqPsn)K%%B8x{ELDAvwYE)4PG@m?^joBMjeDqF z7?;@XOZjxRZ?IA78(x-&gPM7h6xy5Hn&;cdI|rd`dQAGJ{7P$0?kPY> zka&RQU?q<96diY5+(G31NwtIMH|}rJszXci)wDmAJd-lAva}Lp2Xpxdo8`R7O}`?} z?H-~iM>*aul_j?d`UU9CBIH$!N51RMZ+$>oSq5mQo*D0W z6)-nYC|8mz*_Gl-?N1VQg4(ff5B5s=^(XZwW27SN-1n!*XL#D9j0Z4VA6SxmxYjam+Pn+#1bHxS>iANXY*h_B>a z>3!)c_H6>i#!fyUe&x7<_b~M@BHngv z_`W0Jd+p#EX`HaBSvQP+&6sPsV-ZZNi`TfzK%we-}T@4*zRJdq*lCM z{=bb=Qgs?V;Jn;01IwhD)lA_*C~0(Oxq#tIOG*SHE%3i^Q1ddqi*6 zvt2?rdWp>y%ox!u)W{*v>8Tg2FfltM1 z1D?{K&2Np^+Zxy``4YADp(n+@-{nd1spv5}IkK?|9PH#Mp_3sJ6N)@J9`%&`RJ5I} znLOg`SJ9~-M#uhtk0Oq-vvY$7rPCVb-v2Aja#zMfb}m3mc`zi;TU;598Nb_+`q#u- zH-YjJ_@u$pQa|)%NGCw-)f|P}$D9G5s@}mo5XVxvJMq8I3kmxF@AINDLp?9HDK<`m{DKx8-#&LgHF7bfez=RBhpq!zo5x;nfSZ672bZ$ zJlCt~(cm%4k)V@=o0V{HYMC5P-bYXY33Ds{4OgfF0FZt$e&68PoQ!K&t!s^;4RcSikHy`m(Jx?aCk z-Gz6linqx7o2YsUe@mh7!CO#Qqdi!8WTo=R?PGWP{FOYbhG0ZJgnxuLryJgOu8W#2 zJF3=B{M{q!rApJHSQI-FQhIffqEN9qT+u1)*Q$j6OCz<9AOJ<8(Or;-%Ed4AkN~NQ zI(=+}n0QoJ^53qK{11^yJkqP&Fb?C+Y3EZe;o!^E1C_{(sRekq3M<1;tng=G+i^L4 zc`o2+2s#d~I1b+O_#Dl53?Di=eZx;iOG<{4?TeyHa315RZmxj5jCVw_QM6Y-AE9`K0f4%J*}#)1y-x?z0`46$IRdW>=?H)JTOdQ29)D z{0(}FAy&tv@#l8Nv;i1(Q!WYetMp4nA$+=gl`LBipM%-cPAC8f9{aF;+IcD5!S+j& zo(VhdvT#q>oflo6%ZgF3N_*Jwb)_8^!oxaNxChTkcFE8SbUmDCHJb_Nhl)2!3zC%o zklK>h5;c>A-$P~5v1R3jhy;H|2}n?WZkK!3<(itL8%$!DP79BSeRvFVboQ{S7(@_7 z`FcfbfN2#l?LftVVhK%2vZ5&bDS1ri;B_jpL@Ft)rXk&_jJb*HQS3~kgJ~zs?buMc zaFWP+g#$_NLAXvOY?VroTW?pXe9(waPBRSv#|~y8He?m!f$24-H8EU`yVnozpjIT2Ii^OT4zsxw3HA)|OxkMIg*NVYCT zWARQrIr2m!@ay#HJ$SG^NBV&59+*%nYtxW);k0-;%rKIL5^y!olan*E?jcO7iy^E0 zN_KT^>?WB}Cl(1rVrPVqo&uPmdFe()rU5cCLE@h<7e93Fvd8Hb@~LVCYO`7yo-Ez?aJu}`v$}UtESCmbTEB2wNUqWY{xXb3D;j(^-J=#eM zCv>~FvhfQ40m+}Gt@f|2qP|IT`b!gEDF`NSSxMgFOLhd4cdR7uxLFiPuERx+$vKOs z0?AwO1he(!hXXl>z4}bL*S9l}&)(6JwOr?xPJKODU zT1BQIu{Und_sKjQI}K>;?|$ce=X>1maqi`ud(QdM-dn9dXyvr-8wa8i#dlj@YQ4Gr zRp*S8&u!x+?dvGVjDxovukYdH<~B^?n>r znO7D^O{Kx2`E<_IHq{a}mj(CDS99iLQ?1dAe1GSyt{-&$c$CZ7JGCFfr?`w#$eC~s ztYllF#sdHO+e4hOZK_$&nWh>A9F}E0)=t!v3nmJ~e9a*)?=WXNGA)fV*|X#RK^((I zty@s%+-m#lHr~pj?x9HcXsCOX@4n1g*;{S5+J5*Pe58O~C}5+EVK&)s3|LY5K4jd_ zGY15tX(fwFbuFKjJn^A_^W3W!XD<3jcv<-qPyK)ynHllF@V?Bw(kP*H7C~!T(B{KB zfn7&OU=UTEK$)Gq?xc`uUMbjoW6P6b%9x9CPjH4##B@g0x&DfPJ)-t3s6B!)eZ{`{ zreasYK3xCuq>ye4lm^ZrPwV^yI@K$r=lEU09e0xE)u{6vMCwT&q^G4HGZ1p9glB4_ z5MX*=KpJM8g5Da@mxS~s5xsjs??#@Mh^IZ|X^(i0FL;he8T0EnD8bLVxU5!=If#ye znaEK1=rGR=e{8Uz+z!rg0x>6GVWvC&;!pi1UTuRpdbN6{diHcgYhTdX1?yF8J`vIt zM0F5@JJ87M+^`zc<2O}!30mka+-oPkSKJY@l!w*jQ3keKP|7o&sCK)5Ja9guty$34 z2$@+cN4C=JAu(mx7SWf6^rfh*19jrV5a}BU^^Ks5SGc~bNOw)pXGiqqA$>XWoJL(2 zAdG6{;?>Z_t1*hY4sI$RP}C^hC;~{oR|J+QRcRqz*>v-4pT99s1+kkE&E~$W$E7bOy$Q=OdZ*3z_voUcm~m*-h`um{&}hOM+GJXYPpR zI*_xU%RPt8=RVcK7?yRE20S?=CVFTqjb0Z$nP~quUR@FsDK%M9OU>JxWgwnD)`E# zd3mJyY^eDx>KQkH{9F@-b~Tx_3*g~pn8#R zP53lDD1v#bIwS%UFvnpLoC#-0XM_34b$#PKj&J22;o+Y+B>t?h+WG&Sk;yHzx(sg& z83q2%NP#z0;6)7sXo%e65WdAp8a6#eWi*MVlb%_h2Jsfwr;Da^8Gl)e8EnPPR?Jmf zwo<(lSD&?owfTW$UhBe!P6uWV_zMG7!P9pd=c~}M31s;bUiab#R1y}iDc3AC->I(z%h_xZ0%TqKrx;4|4=ynSZ58S z2Z1@(M8Ru7MbI8G)-D)pg>9AEu`FaPTQKg97F7g?BA%lm&r!j1 zQt<46-R^0cPk$g4+1u$^HJCD*T~Bi${nooIrOfs~|64h+1eOn%?Ty$?ar$0S z)C4j1;6Y^aZ0Ndd>s@Q9qB39%z&N)H1yztMoH_X1t-5Q=Z7<6ym3F$~&sznpZsoLO z^Jnt&pM#T(MAWITf1M4>+z)-7@b}9v?cNe^ zgPnROvq9>uPm#`Ba`EM#lI-N-X7J^%Nah8aUm21y^s8(gxwr}P*GbtJ`gJ~m{-tU+ zhTf~v;|ogl64Fp!76un6Lc{5t7~&I+h7DfQ>ifA?jEyJ=jgT$DkWiS8FZU7^7j>x^ zTFl#ieVb%S?log5T0km%P)*AIRpu?6v`hZYm51@3HHBL zUM1QG8h?G5@cLQkPoc8Fmf5j53Mdc2=X&J7b8E*BgoI}1i7+XiAg?B zAWl8K>&$t}NfUG4$`o(COf^?8lH(@l z>QeyDl+V>0NHcQ{cCXUAr&9csWED$Y#qj^ZEF?EJxKV{qjbIag^|z|X17A^3J}8Dy zhMDIXT|)gef&Y3dfW~I6n1ffg#_$%k;qa`X)PIX31 zB_UHu&=zs;3%U2rw?q!~gbwuZlHO>FX*QipaUzNHS<(60Ouh#51~Uo6Nc^5RkCrt@ zqnEzUDXNrB zQLULof`i!S&-e3FhL@Ic^t>1 zo-jbHaB%F{izA1x2Jv=4EDvT44*|LyAWg@ZHP=KAc64-XA{A(Q+3vC4!33OweQ2yVv) zoRgQhHh`tP#0{aUrJAT(KYcD@C=40EF^r8<7a>{E=TW8h<|&k70Us+sAyXmX!v%gL zO11r1{CA4KRX_yJ57f*f@QonVJoK4h0e)F&+QU$9Gnz()_-#veqg4EkR0?@`65OCb zegX+a(Kul!o0KT%zSVeOFUh?3@RP?R^Z1j8?~%;i$3Oi!$^87MkMG(f6Q$jK-Qeng z^|?SK-hIJ=xB9vUxSw@hDRpgtoxZ^d=O(H^sP?hZ-q8!r;md3QZejxTJZ!(~ira;i zf^c9!-virwV8}PdUMI?BTk?b&)4&NG02P-H_oRg2tYa*M&+$P!6Pl6HzVQqFK35`2 z4@Q|*QP3aJwG6P`mj<6If#W#P53XM?kN5k=aEWoX4)ij9VXW)iYWrU;7-PXhEYz@x z4j!}}TTJfi>Kp9^ZO%uqr56==X#Lm5E)4as_;rE33s^Q6=ZK>W%=sW z=~~xqBveGICwNeff~u3nUuG??;u6e`t;3Dr6TX;?b#S1m0v~3H)_O<#0BE~@ajg4V zBaRWwnsB#=a1CtU%3^jvIKiT#?Ya*X_Knv-O@?2468ffZ<8l@!={R`_CkJuz7AE;2 zPJRT*qP+21Z~rB{84_%UsHohy7^|=nag4~jPZtJZi$76E5yjN&82lTY{3A~8;$&S1 z_Fou;KZKvCyqaZAM`&aeYfT0v}B5NG?V0uy{*bxb6e77FIbNv7QhM)!YnG9~iQpB^P&<_Z(oUP_+fCnTL8l5wF zEJBniCX;GJH&ijDOa~ydY@liRv<"$tmp/fifo-input.out" 2>&1 & +fifo_pid=$! +( sleep 20; kill -9 "$fifo_pid" 2>/dev/null ) & +fifo_watchdog=$! +if wait "$fifo_pid"; then fail fifo-input; fi +kill "$fifo_watchdog" 2>/dev/null || true +if ! grep -Fq 'delivery replay: input is not a regular file' "$tmp/fifo-input.out" || + grep -Fq Traceback "$tmp/fifo-input.out"; then + fail fifo-input-error +fi printf '\377' >"$tmp/reconcile-state/invalid-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \