From 7132fb7233239587d23f3a251d142d8da0ff5cc2 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:10:37 +0100 Subject: [PATCH 1/4] fix(recovery): verify current checkpoint custody --- docs/durable-checkpoints.md | 15 +- .../devflow_temporal/delivery_checkpoints.py | 72 +++- .../test_delivery_checkpoint_native_escape.py | 148 ++++++++ runtime/tests/test_delivery_checkpoints.py | 338 +++++++++++++++++- 4 files changed, 567 insertions(+), 6 deletions(-) create mode 100644 runtime/tests/test_delivery_checkpoint_native_escape.py diff --git a/docs/durable-checkpoints.md b/docs/durable-checkpoints.md index 4169679e..3d46578f 100644 --- a/docs/durable-checkpoints.md +++ b/docs/durable-checkpoints.md @@ -93,6 +93,16 @@ The existing loopback API and CSRF boundary expose these operations: - `POST /api/runs/{run_id}/checkpoints/{checkpoint_id}/restore` accepts a new `command_id` and `expected_revision`. +Readback preserves the original `acknowledged_resumable` value as history. Its +current `resumable` observation also compares the admitted authority, run and +cumulative budget, issue owner, independent Git owner and candidate branch, +original resource manifest, and the current writer witness. If the checkout is +still present, it compares its retained source and index under that witness; +an absent checkout can still be restored from the sealed artifacts. Changed or +unavailable proof appears in `resume_blockers`. This is a read-only observation, +not a reservation or permission to resume: restoration must reacquire custody +and repeat its own admission checks. + Creation requires a stopped run, reconciled external effects, closed original launch and role leases, proven registered workspace ownership, and a fresh check for open workspace actors. Registered native journals currently cause a typed @@ -210,7 +220,10 @@ publication only when current fenced source/state reproduces its exact manifest and every artifact. Corrupt, incomplete, foreign or inconsistent bytes still refuse. Resource pressure or an interrupted write cannot produce a successful checkpoint receipt. Checkpoint directories live outside disposable -worktrees and have no automatic expiry or cleanup path. +worktrees and have no automatic expiry or cleanup path. Explicit retained-resource +pruning preserves checkpoint directories and durable session/evidence material; +a blocked candidate remains retained. No checkpoint expiration operation is +currently provided. Restoration validates every artifact and current execution authority, budget, owner, original resource generation and native closure again. It reconstructs a diff --git a/runtime/src/devflow_temporal/delivery_checkpoints.py b/runtime/src/devflow_temporal/delivery_checkpoints.py index 4712c5e3..324e4b75 100644 --- a/runtime/src/devflow_temporal/delivery_checkpoints.py +++ b/runtime/src/devflow_temporal/delivery_checkpoints.py @@ -1423,11 +1423,41 @@ def read_checkpoint(store, run_id, checkpoint_id): spec = store.effective_spec(run_id) with store._connect() as db: value, receipt = _read_acknowledged(db, spec, checkpoint_id) - current_effects = _frozen(db, _row(db, run_id))["delivery_effects"] + # Keep the tracking snapshot coherent without holding a SQL read + # transaction while the potentially large artifact inventory is hashed. + db.execute("BEGIN") + row = _row(db, run_id) + current = _frozen(db, row) + claim = store.state.claim_for(db, row["work_id"]) + owner = dict(claim) if claim else None + db.commit() # Readback retains the original ACK, but does not advertise a native # checkpoint as resumable under today's admission contract. Current custody # can have changed since the ACK; inability to inspect it is also unknown. blockers = list(value.get("resume_blockers", [])) + if digest(spec) != value["spec_sha256"]: + blockers.append({"scope": "authority", "reason": "checkpoint_authority_changed"}) + if not store.owns_execution(spec): + blockers.append({"scope": "authority", "reason": "checkpoint_transport_ownership_unknown"}) + if not _same_snapshot(current, value): + blockers.append({"scope": "run", "reason": "checkpoint_run_state_changed"}) + if owner != value["owner"]: + blockers.append({"scope": "owner", "reason": "checkpoint_execution_owner_changed"}) + try: + from .delivery_workspace_custody import source_for + + source = source_for(spec) + except (OSError, ValueError, subprocess.SubprocessError): + blockers.append({"scope": "source", "reason": "checkpoint_workspace_owner_unknown"}) + else: + try: + head = _git(source, "rev-parse", "refs/heads/" + value["source"]["branch"]) + except (OSError, ValueError, subprocess.SubprocessError): + blockers.append({"scope": "source", "reason": "checkpoint_candidate_branch_unknown"}) + else: + if head.decode().strip() != value["source"]["candidate"]["head"]: + blockers.append({"scope": "source", + "reason": "checkpoint_candidate_branch_changed"}) effect_blockers = [] feature_unknown = None current_feature = None @@ -1446,7 +1476,7 @@ def read_checkpoint(store, run_id, checkpoint_id): feature_unknown = "registry_observation_unknown" for scope, effects in ( ("delivery", value["frozen"]["delivery_effects"]), - ("delivery", current_effects), + ("delivery", current["delivery_effects"]), ("feature", (value.get("execution_ledger") or {}).get("execution_effects", [])), ("feature", (current_feature or {}).get("execution_effects", [])), ): @@ -1459,14 +1489,49 @@ def read_checkpoint(store, run_id, checkpoint_id): effect_blockers.append(blocker) if blocker not in blockers: blockers.append(blocker) + retained_source_unknown = False try: registry = RunResources(spec, read_only=True) with registry.locked(read_only=True) as resources: custody_unknown = bool(value.get("closed_journals") or resources["processes"]) + entry = resources["roots"].get(spec["checkout"]) + resource_changed = ( + not entry + or entry.get("kind") != "checkout" + or entry.get("identity") != value["checkout_identity"] + or _hash(read_private(registry.manifest, raw=True)) + != value["resource_manifest_sha256"] + ) + try: + # Follow creation/restoration's resource-before-writer lock order. + # The exclusive witness is observational and grants no later work. + with store._checkpoint_writer_context(spec): + writer_unknown = False + checkout = Path(spec["checkout"]) + if os.path.lexists(checkout) and _index_baseline_supported(value): + try: + from .delivery_retained_checkpoint import verify_retained_source + + _openers(checkout) + verify_retained_source( + spec, value, _root(spec) / checkpoint_id, resources) + except (OSError, ValueError, KeyError, sqlite3.Error, + subprocess.SubprocessError): + retained_source_unknown = True + except (OSError, ValueError, sqlite3.Error, subprocess.SubprocessError): + writer_unknown = True except (OSError, ValueError, KeyError): custody_unknown = True + resource_changed = True + writer_unknown = True if custody_unknown: blockers.append({"scope": "native", "reason": NativeCheckpointCustodyUnknown.code}) + if resource_changed: + blockers.append({"scope": "resource", "reason": "checkpoint_resource_custody_unknown"}) + if writer_unknown: + blockers.append({"scope": "writer", "reason": "checkpoint_writer_custody_unknown"}) + if retained_source_unknown: + blockers.append({"scope": "source", "reason": "checkpoint_retained_source_unverified"}) if feature_unknown: blockers.append({"scope": "feature", "reason": feature_unknown}) index_unknown = not _index_baseline_supported(value) @@ -1475,8 +1540,7 @@ def read_checkpoint(store, run_id, checkpoint_id): return { **receipt, "acknowledged_resumable": receipt["resumable"], - "resumable": (receipt["resumable"] and not custody_unknown - and not effect_blockers and not feature_unknown and not index_unknown), + "resumable": receipt["resumable"] and not blockers, "resume_blockers": blockers, "authority_sha256": value["spec_sha256"], "owner": value["owner"], diff --git a/runtime/tests/test_delivery_checkpoint_native_escape.py b/runtime/tests/test_delivery_checkpoint_native_escape.py new file mode 100644 index 00000000..de91ea46 --- /dev/null +++ b/runtime/tests/test_delivery_checkpoint_native_escape.py @@ -0,0 +1,148 @@ +"""An observed native monitor cannot prove every descendant has settled.""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest +from temporal_test_server import local_temporal +from temporalio import workflow +from temporalio.worker import UnsandboxedWorkflowRunner, Worker +from test_delivery_api import api_fixture as api_fixture +from test_native_interrupted_gate import native_gates + +from devflow_temporal.delivery_api import _dispatch_start, create_app +from devflow_temporal.delivery_native_process import NativeProcess, process_table +from devflow_temporal.delivery_resources import read_private +from devflow_temporal.delivery_writer_custody import to_thread + + +@workflow.defn(name="DevflowDeliveryWorkflow") +class NativeCheckpointOwner: + @workflow.run + async def run(self, spec, recovery=None): + return {"run_id": spec["run_id"], "execution_state": "blocked", + "outcome": "blocked", "iteration": 0} + + +@pytest.mark.asyncio +async def test_public_native_checkpoint_refuses_escaped_post_go_descendant( + api_fixture, tmp_path, +): + async with local_temporal() as env: + config_path, _request = api_fixture + raw = json.loads(config_path.read_text()) + raw["temporal_address"] = env.client.service_client.config.target_host + config_path.write_text(json.dumps(raw)) + fixture = native_gates.__wrapped__(api_fixture, tmp_path) + g = next(fixture) + ready, release = tmp_path / "escaped-ready.json", tmp_path / "escaped-release" + actor = None + try: + # The owned native body exits after a fast double fork. Its child + # closes inherited witnesses and stdio, reparents, and keeps no + # checkout cwd or open checkout file for lsof to observe. + code = """import json,os,sys,time +from pathlib import Path +from devflow_temporal.delivery_native_process import process_table +if os.fork(): os._exit(0) +os.setsid() +if os.fork(): os._exit(0) +os.closerange(0,1024) +os.chdir('/') +pid=os.getpid() +Path(sys.argv[1]).write_text(json.dumps({'pid':pid,'identity':process_table()[pid]['identity']})) +while not Path(sys.argv[2]).exists(): time.sleep(.02) +""" + folder = Path(g.spec["state_dir"]) / "owned-opaque-check" + process = NativeProcess( + g.spec, folder, argv=[sys.executable, "-I", "-c", code, str(ready), str(release)], + cwd=g.broker.checkout, timeout=20, + environment={"PATH": os.environ["PATH"], + "HOME": str(tmp_path / "owned-provider-home")}) + observed = await to_thread(g.spec, process.run) + deadline = time.monotonic() + 10 + while not ready.exists(): + assert time.monotonic() < deadline + await asyncio.sleep(.02) + actor = json.loads(ready.read_text()) + journal = read_private(folder / "native-process.json") + assert observed["cleanup"] == "observed-native-confirmed" + assert journal["monitoring_complete"] is True + assert str(actor["pid"]) not in journal["owned"] + assert process_table()[actor["pid"]]["identity"] == actor["identity"] + + # The exact original SDK execution closes; no mocked controller + # response or caller-provided custody assertion is used. + submitted = g.store.submitted_spec(g.spec["run_id"]) + async with Worker( + env.client, task_queue=g.store.config.queue, + workflow_runner=UnsandboxedWorkflowRunner(), workflows=[NativeCheckpointOwner], + ): + await _dispatch_start(SimpleNamespace(store=g.store, config=g.store.config), + env.client, g.store.pending_starts()[0], submitted) + assert (await env.client.get_workflow_handle( + "delivery-" + g.spec["run_id"]).result())["outcome"] == "blocked" + with g.store._connect() as db: + db.execute("UPDATE delivery_runs SET phase='blocked',execution_state='blocked'," + "outcome='blocked' WHERE run_id=?", (g.spec["run_id"],)) + revision = db.execute("SELECT revision FROM delivery_runs WHERE run_id=?", + (g.spec["run_id"],)).fetchone()[0] + url = g.store.config.dashboard_url.rstrip("/") + app = create_app(g.store.config.path) + path = "/api/runs/" + g.spec["run_id"] + "/checkpoints" + command = {"command_id": "escaped-native", "expected_revision": revision} + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url=url, + ) as client: + session = (await client.get("/api/session")).json() + headers = {"origin": url, "x-devflow-csrf": session["csrf_token"]} + response = await client.post(path, json=command, headers=headers) + assert response.status_code == 409, response.text + assert response.json()["detail"]["code"] == "native_checkpoint_custody_unknown" + assert process_table()[actor["pid"]]["identity"] == actor["identity"] + + # Even later disappearance cannot manufacture complete closure. + release.touch() + deadline = time.monotonic() + 10 + while True: + current = process_table().get(actor["pid"], {}) + if (current.get("identity") != actor["identity"] + or current.get("stat", "Z").startswith("Z")): + break + assert time.monotonic() < deadline + await asyncio.sleep(.02) + retry = await client.post(path, json=command, headers=headers) + assert retry.status_code == 409 + assert retry.json()["detail"]["code"] == "native_checkpoint_custody_unknown" + with g.store._connect() as db: + assert db.execute("SELECT 1 FROM delivery_commands WHERE command_id=?", + ("checkpoint-create:escaped-native",)).fetchone() is None + assert not list((Path(g.spec["state_dir"]) / "checkpoints").glob("*/manifest.json")) + finally: + release.touch() + if actor: + current = process_table().get(actor["pid"], {}) + if (current.get("identity") == actor["identity"] + and not current.get("stat", "Z").startswith("Z")): + try: + os.kill(actor["pid"], signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + 5 + while True: + current = process_table().get(actor["pid"], {}) + if (current.get("identity") != actor["identity"] + or current.get("stat", "Z").startswith("Z")): + break + assert time.monotonic() < deadline + await asyncio.sleep(.02) + next(fixture, None) diff --git a/runtime/tests/test_delivery_checkpoints.py b/runtime/tests/test_delivery_checkpoints.py index 65a4b691..13c7d8e9 100644 --- a/runtime/tests/test_delivery_checkpoints.py +++ b/runtime/tests/test_delivery_checkpoints.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import multiprocessing import os @@ -9,7 +10,7 @@ import sqlite3 import subprocess import sys -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from pathlib import Path import httpx @@ -521,7 +522,55 @@ def test_selected_session_bytes_survive_source_deletion(stopped): @pytest.mark.asyncio async def test_public_checkpoint_routes_require_csrf_and_return_verified_readback(stopped): + from devflow_temporal.contracts import digest + from devflow_temporal.delivery_continuation import session_state_digest + store, spec, broker, row = stopped + # A passed fake implementation has an unpublished commit, then separate + # staged, unstaged and untracked work when a check stops the delivery. + git(broker.checkout, "commit", "-qm", "Owned unpublished implementation") + (broker.checkout / "README.md").write_text("staged after implementation\n") + git(broker.checkout, "add", "README.md") + (broker.checkout / "README.md").write_text("staged plus unfinished work\n") + candidate = broker.candidate() + staged = git(broker.checkout, "diff", "--cached") + assert candidate["head"] != spec["base_sha"] + assert not git(Path(spec["origin_url"]), "for-each-ref", "refs/heads/" + spec["branch"]) + + state = Path(spec["state_dir"]) + session_id = "owned-implementation-session" + home = state / "role-homes/implement" + sessions = home / "codex/sessions" + sessions.mkdir(parents=True, mode=0o700) + transcript = sessions / (session_id + ".jsonl") + transcript.write_text('{"fixture":"implementation transcript"}\n') + transcript.chmod(0o600) + (home / "codex/auth.json").write_text("private fixture credential") + session_digest = session_state_digest(home, session_id) + evidence = state / "evidence" / "stopped-check.xml" + evidence.parent.mkdir(mode=0o700) + evidence.write_text("\n") + evidence.chmod(0o600) + evidence_sha256 = hashlib.sha256(evidence.read_bytes()).hexdigest() + job_key = digest({"owned": "passed-implementation"}) + request = {"spec": spec, "role": "implement", "iteration": 1, "candidate": candidate} + write_private(state / "attempts" / job_key / "request.json", request) + result = {"role": "implement", "iteration": 1, "status": "pass", "candidate": candidate, + "session_id": session_id, "cleanup": "confirmed"} + checks = {"prepublish": {"state": "failed", "candidate_id": candidate["id"], + "results": [{"id": "owned-check", "passed": False, + "cleanup": "confirmed", "path": str(evidence), + "sha256": evidence_sha256}]}} + with store._connect() as db: + db.execute("INSERT INTO delivery_attempts " + "(job_key,run_id,role,iteration,candidate_id,state,session_id,result_json," + "cleanup) VALUES (?,?,'implement',1,?,'finished',?,?,'confirmed')", + (job_key, spec["run_id"], candidate["id"], session_id, json.dumps(result))) + db.execute("UPDATE delivery_runs SET candidate_json=?,checks_json=? WHERE run_id=?", + (json.dumps(candidate), json.dumps(checks), spec["run_id"])) + row = dict(db.execute("SELECT * FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone()) + claim = dict(store.state.claim_for(db, spec["work_id"])) app = create_app(store.config.path) url = store.config.dashboard_url.rstrip("/") async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=url) as client: @@ -534,8 +583,12 @@ async def test_public_checkpoint_routes_require_csrf_and_return_verified_readbac assert response.status_code == 200, response.text receipt = response.json() assert (await client.get(path + "/" + receipt["checkpoint_id"])).json()["resumable"] + checkpoint = state / "checkpoints" / receipt["checkpoint_id"] + assert not list(checkpoint.rglob("auth.json")) shutil.rmtree(Path(spec["source_path"])) shutil.rmtree(broker.checkout) + shutil.rmtree(home) + evidence.unlink() response = await client.post( path + "/" + receipt["checkpoint_id"] + "/restore", json={"command_id": "http-restore", "expected_revision": row["revision"]}, @@ -543,6 +596,289 @@ async def test_public_checkpoint_routes_require_csrf_and_return_verified_readbac ) assert response.status_code == 200, response.text assert response.json()["execution_started"] is False + assert response.json()["candidate"] == candidate_for(broker.checkout) + assert broker.candidate() == candidate + assert git(broker.checkout, "diff", "--cached") == staged + assert (broker.checkout / "README.md").read_text() == "staged plus unfinished work\n" + assert (broker.checkout / "owned-untracked.txt").read_text() == "recover me\n" + assert session_state_digest(home, session_id) == session_digest + assert hashlib.sha256(evidence.read_bytes()).hexdigest() == evidence_sha256 + assert not (home / "codex/auth.json").exists() + with store._connect() as db: + assert dict(db.execute("SELECT * FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone()) == row + assert dict(store.state.claim_for(db, spec["work_id"])) == claim + + +@pytest.mark.asyncio +async def test_public_recovery_continues_restored_stopped_gate(stopped, monkeypatch): + """Recovery must use the same stopped candidate after both fixture checkouts vanish.""" + from devflow_temporal.contracts import digest + from devflow_temporal.delivery_resource_maintenance import prune + + store, spec, broker, _row = stopped + # Keep this continuation inside its admitted source scope. The fake role and + # closed workflow are fixtures; source, cleanup, checkpoint and public + # recovery admission use their actual implementations. + (broker.checkout / "owned-untracked.txt").unlink() + (broker.checkout / ".gitignore").unlink() + (broker.checkout / "auth.json").unlink() + git(broker.checkout, "commit", "-qm", "Fixture unpublished implementation") + (broker.checkout / "README.md").write_text("Unfinished implementation\n") + candidate = broker.candidate() + state_dir = Path(spec["state_dir"]) + session_id = "owned-implementation-session" + home = state_dir / "role-homes/implement" + sessions = home / "codex/sessions" + sessions.mkdir(mode=0o700, parents=True) + transcript = sessions / (session_id + ".jsonl") + transcript.write_text('{"owned":"fixture"}\n') + transcript.chmod(0o600) + job_key = digest({"owned": "implementation"}) + request = {"spec": spec, "role": "implement", "iteration": 1, "candidate": candidate} + write_private(state_dir / "attempts" / job_key / "request.json", request) + implementation = {"role": "implement", "iteration": 1, "status": "pass", + "session_id": session_id, "cleanup": "confirmed", "candidate": candidate} + cleanup = RunResources(spec).finalize("blocked") + assert cleanup["state"] == "confirmed", cleanup + checks = {"prepublish": {"state": "failed", "source_unchanged": True, + "candidate_id": candidate["id"], "results": [ + {"id": "packaging", "passed": False, + "cleanup": "confirmed", "exit_code": 1}]}, + "resource_cleanup": cleanup} + state = {"run_id": spec["run_id"], "revision": 9, "iteration": 1, "phase": "blocked", + "execution_state": "blocked", "outcome": "blocked", "cleanup": "confirmed", + "error": "prepublication repair limit exhausted", "candidate": candidate, + "pull_request": None, "candidate_revision": 2, "roles": [implementation], + "checks": checks, "usage": {"tokens": 419, "wall_seconds": 12}, + "findings": ["Historical packaging failure"]} + store.project(spec["run_id"], phase="blocked", execution_state="blocked", + event_type="blocked", message=state["error"], candidate=candidate, + checks=checks, usage=state["usage"], iteration=1, protocol_revision=9, + outcome="blocked", cleanup="confirmed", error=state["error"]) + with store._connect() as db: + db.execute("INSERT INTO delivery_attempts " + "(job_key,run_id,role,iteration,candidate_id,state,session_id,result_json," + "cleanup) VALUES (?,?,'implement',1,?,'finished',?,?,'confirmed')", + (job_key, spec["run_id"], candidate["id"], session_id, + json.dumps(implementation))) + db.execute("UPDATE delivery_runs SET workflow_id=? WHERE run_id=?", + ("delivery-" + spec["run_id"], spec["run_id"])) + store.state.release_work(db, spec["work_id"], "external:devflow:" + spec["run_id"]) + row = dict(db.execute("SELECT * FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone()) + closed = {"workflow_id": "delivery-" + spec["run_id"], "execution_run_id": "closed-original", + "request_digest": spec["request_digest"], "recovery_digest": None, + "result": state, "closed_at": "2026-01-01T00:00:00+00:00"} + monkeypatch.setattr(store, "_completed_temporal_result", lambda *_args, **_kw: closed) + monkeypatch.setattr(store, "_ensure_no_remote_pr", lambda *_: None) + app = create_app(store.config.path) + app.state.delivery.store = store + url = store.config.dashboard_url.rstrip("/") + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=url) as client: + headers = {"origin": url, "x-devflow-csrf": (await client.get("/api/session")).json()[ + "csrf_token"]} + path = "/api/runs/" + spec["run_id"] + "/checkpoints" + saved = await client.post(path, headers=headers, json={ + "command_id": "integration-save", "expected_revision": row["revision"]}) + assert saved.status_code == 200, saved.text + pruned = prune(store, spec["run_id"]) + assert any(item["path"] == str(broker.checkout) and item["state"] == "retained" + for item in pruned["roots"]) + retained = await client.get(path + "/" + saved.json()["checkpoint_id"]) + assert retained.status_code == 200 and retained.json()["resumable"] + assert retained.json()["manifest_sha256"] == saved.json()["manifest_sha256"] + shutil.rmtree(Path(spec["source_path"])) + shutil.rmtree(broker.checkout) + shutil.rmtree(home) + restored = await client.post(path + "/" + saved.json()["checkpoint_id"] + "/restore", + headers=headers, json={"command_id": "integration-restore", + "expected_revision": row["revision"]}) + assert restored.status_code == 200, restored.text + assert broker.candidate() == candidate and transcript.read_text() == '{"owned":"fixture"}\n' + command = {"continuation_kind": "stopped_delivery_resume", "command_id": "resume-1", + "expected_revision": 9, "expected_iteration": 1, + "expected_candidate_id": candidate["id"], + "expected_candidate_head": candidate["head"], "additional_iterations": 1} + admitted = await client.post("/api/runs/" + spec["run_id"] + "/continue-repair", + headers=headers, json=command) + assert admitted.status_code == 200, admitted.text + assert admitted.json()["run_id"] == spec["run_id"] + assert admitted.json()["authorized_through_iteration"] == 2 + replay = await client.post("/api/runs/" + spec["run_id"] + "/continue-repair", + headers=headers, json=command) + assert replay.status_code == 200 and replay.json() == admitted.json() + with store._connect() as db: + current = db.execute("SELECT * FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone() + recovery = json.loads(current["recovery_json"]) + assert recovery["session_id"] == session_id + assert recovery["state"]["checks"] == checks + assert recovery["state"]["usage"] == state["usage"] + assert recovery["candidate"] == candidate + assert db.execute("SELECT count(*) FROM delivery_runs").fetchone()[0] == 1 + assert store.state.claim_for(db, spec["work_id"])["owner"] == ( + "external:devflow:" + spec["run_id"]) + assert not db.execute("SELECT 1 FROM delivery_effects WHERE kind='publish'").fetchone() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drift", ["budget", "authority", "owner", "workspace_owner", "resource", + "writer", "branch", "source", "index"]) +async def test_public_readback_refuses_stale_custody_without_rewriting_ack(stopped, drift): + """A historical ACK must not advertise recovery after its current proof diverges.""" + from devflow_temporal.delivery_writer_custody import shared + + store, spec, broker, row = stopped + if drift == "branch": + git(broker.checkout, "commit", "-qm", "Owned unpublished fixture") + app = create_app(store.config.path) + app.state.delivery.store = store + url = store.config.dashboard_url.rstrip("/") + path = "/api/runs/" + spec["run_id"] + "/checkpoints" + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=url) as client: + token = (await client.get("/api/session")).json()["csrf_token"] + headers = {"origin": url, "x-devflow-csrf": token} + saved = await client.post(path, headers=headers, json={ + "command_id": "current-custody-save", "expected_revision": row["revision"]}) + assert saved.status_code == 200, saved.text + receipt = saved.json() + checkpoint_path = path + "/" + receipt["checkpoint_id"] + assert (await client.get(checkpoint_path)).json()["resumable"] + if drift == "budget": + with store._connect() as db: + db.execute("UPDATE delivery_runs SET usage_json=? WHERE run_id=?", + ('{"tokens":420}', spec["run_id"])) + reason = "checkpoint_run_state_changed" + elif drift == "authority": + with store._connect() as db: + db.execute("UPDATE delivery_runs SET accepted_plan_text=? WHERE run_id=?", + ("A different accepted plan", spec["run_id"])) + reason = "checkpoint_authority_changed" + elif drift == "owner": + with store._connect() as db: + store.state.release_work(db, spec["work_id"], "external:devflow:" + spec["run_id"]) + reason = "checkpoint_execution_owner_changed" + elif drift == "workspace_owner": + owner = Path(spec["state_dir"]) / "workspace-owner.git" + owner.rename(owner.with_name("fixture-owner-unavailable.git")) + reason = "checkpoint_workspace_owner_unknown" + elif drift == "resource": + registry = RunResources(spec, read_only=True) + resources = read_private(registry.manifest) + resources["roots"][str(broker.checkout)]["generation"] = 42 + write_private(registry.manifest, resources) + reason = "checkpoint_resource_custody_unknown" + elif drift == "branch": + git(broker.checkout, "update-ref", "refs/heads/" + spec["branch"], spec["base_sha"]) + reason = "checkpoint_candidate_branch_changed" + elif drift == "source": + (broker.checkout / "README.md").write_text("new unsaved stopped progress\n") + reason = "checkpoint_retained_source_unverified" + elif drift == "index": + git(broker.checkout, "update-index", "--assume-unchanged", "README.md") + reason = "checkpoint_retained_source_unverified" + else: + reason = "checkpoint_writer_custody_unknown" + + with shared(spec) if drift == "writer" else nullcontext(): + if drift == "writer": + (broker.checkout / "README.md").write_text("new unsaved writer progress\n") + observed = await client.get(checkpoint_path) + assert observed.status_code == 200, observed.text + readback = observed.json() + assert readback["acknowledged_resumable"] is True + assert readback["resumable"] is False + assert reason in {item["reason"] for item in readback["resume_blockers"]} + assert readback["manifest_sha256"] == receipt["manifest_sha256"] + if drift == "writer": + refused = await client.post(path, headers=headers, json={ + "command_id": "live-writer", "expected_revision": row["revision"]}) + assert refused.status_code == 409 + if drift == "writer": + assert (broker.checkout / "README.md").read_text() == "new unsaved writer progress\n" + return + if drift in {"source", "index"}: + assert broker.checkout.exists() + if drift == "source": + assert (broker.checkout / "README.md").read_text() == ( + "new unsaved stopped progress\n") + return + shutil.rmtree(broker.checkout) + restored = await client.post(checkpoint_path + "/restore", headers=headers, json={ + "command_id": "stale-custody-restore", "expected_revision": row["revision"]}) + assert restored.status_code == 409 + assert not broker.checkout.exists() + with store._connect() as db: + assert db.execute( + "SELECT count(*) FROM delivery_commands WHERE command_id=?", + ("checkpoint-restore:stale-custody-restore",), + ).fetchone()[0] == 0 + + +@pytest.mark.asyncio +async def test_public_historical_checkpoint_refuses_new_native_custody(stopped): + """An older ACK remains evidence when current native closure is unknown.""" + store, spec, broker, row = stopped + receipt = create_checkpoint(store, spec["run_id"], "before-native", row["revision"]) + registry = RunResources(spec, read_only=True) + resources = read_private(registry.manifest) + # Model a newly registered native journal after an existing acknowledgement. + resources["processes"].append(str(Path(spec["state_dir"]) / "native-process.json")) + write_private(registry.manifest, resources) + app = create_app(store.config.path) + url = store.config.dashboard_url.rstrip("/") + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=url) as client: + path = "/api/runs/" + spec["run_id"] + "/checkpoints" + token = (await client.get("/api/session")).json()["csrf_token"] + headers = {"origin": url, "x-devflow-csrf": token} + readback = (await client.get(path + "/" + receipt["checkpoint_id"])).json() + assert readback["acknowledged_resumable"] is True + assert readback["resumable"] is False + assert "native_checkpoint_custody_unknown" in { + item["reason"] for item in readback["resume_blockers"]} + shutil.rmtree(broker.checkout) + response = await client.post(path + "/" + receipt["checkpoint_id"] + "/restore", + headers=headers, json={"command_id": "native-restore", + "expected_revision": row["revision"]}) + assert response.status_code == 409 + assert response.json()["detail"]["code"] == "native_checkpoint_custody_unknown" + assert not broker.checkout.exists() + assert not list(broker.checkout.parent.glob(".checkpoint-restore-*")) + with store._connect() as db: + assert db.execute("SELECT 1 FROM delivery_mutations WHERE command_id=?", + ("checkpoint-restore:native-restore",)).fetchone() is None + + +def test_fenced_checkpoint_preserves_unknown_effect_without_restoration(stopped): + """A trusted fence can retain an uncertain effect but cannot replay it.""" + store, spec, broker, row = stopped + with store._connect() as db: + db.execute("INSERT INTO delivery_effects VALUES (?,?,?,?,'pending',NULL,'now')", + ("unknown-publish", spec["run_id"], "publish", "{}")) + fence = durable_fence(store, spec) + with pytest.raises(ValueError, match="authenticated pause fence"): + create_checkpoint(store, spec["run_id"], "unknown-without-fence", row["revision"], + allow_unresolved_effects=True) + receipt = create_checkpoint(store, spec["run_id"], "unknown-with-fence", row["revision"], + fence=fence, allow_unresolved_effects=True) + assert not receipt["resumable"] + assert receipt["resume_blockers"] == [{"scope": "delivery", + "reason": "external_outcome_unknown", "effect_key": "unknown-publish", + "kind": "publish", "state": "pending"}] + readback = read_checkpoint(store, spec["run_id"], receipt["checkpoint_id"]) + assert not readback["resumable"] and readback["pending_effects"] + shutil.rmtree(broker.checkout) + with pytest.raises(ValueError, match="require reconciliation"): + restore_checkpoint(store, spec["run_id"], receipt["checkpoint_id"], "unknown-restore", + row["revision"], fence=fence) + assert not broker.checkout.exists() + with store._connect() as db: + effect = db.execute("SELECT state,observed_json FROM delivery_effects " + "WHERE effect_key='unknown-publish'").fetchone() + assert tuple(effect) == ("pending", None) + assert db.execute("SELECT 1 FROM delivery_mutations WHERE command_id=?", + ("checkpoint-restore:unknown-restore",)).fetchone() is None @pytest.mark.asyncio From efb7a5dc51f1a9f670f54fea00d2c030533bff22 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 20:56:43 +0100 Subject: [PATCH 2/4] fix(recovery): preserve fake provider continuation --- .../devflow_temporal/delivery_checkpoints.py | 6 +- .../devflow_temporal/delivery_continuation.py | 10 +- runtime/src/devflow_temporal/supervisor.py | 5 + .../test_delivery_checkpoint_fake_temporal.py | 228 ++++++++++++++++++ 4 files changed, 246 insertions(+), 3 deletions(-) create mode 100644 runtime/tests/test_delivery_checkpoint_fake_temporal.py diff --git a/runtime/src/devflow_temporal/delivery_checkpoints.py b/runtime/src/devflow_temporal/delivery_checkpoints.py index 324e4b75..caebe3b2 100644 --- a/runtime/src/devflow_temporal/delivery_checkpoints.py +++ b/runtime/src/devflow_temporal/delivery_checkpoints.py @@ -875,7 +875,11 @@ def references(value): if not home.is_relative_to(state): raise ValueError("checkpoint session home escaped its original run") _openers(home) - paths = session_state_paths(home, attempt["session_id"]) + # The fixed fake provider has a synthetic session ID but writes no + # transcript. Preserve fixture transcripts when present; native + # providers must always supply their required session. + paths = session_state_paths( + home, attempt["session_id"], allow_absent=spec["provider"] == "fake") selected_manifest(home, paths) # Private owned regular files, no linked ancestors. selected.update((home / item).relative_to(state).as_posix() for item in paths) elif attempt["session_id"] and spec["provider"] != "fake": diff --git a/runtime/src/devflow_temporal/delivery_continuation.py b/runtime/src/devflow_temporal/delivery_continuation.py index 6c225762..a856325a 100644 --- a/runtime/src/devflow_temporal/delivery_continuation.py +++ b/runtime/src/devflow_temporal/delivery_continuation.py @@ -67,14 +67,18 @@ def selected_digest(root: Path, paths: list[str]) -> str: ) -def session_state_paths(role_home: Path, session_id: str) -> list[str]: - """Select only provider session data; auth, profile, caches and tools are regenerated.""" +def session_state_paths( + role_home: Path, session_id: str, *, allow_absent: bool = False, +) -> list[str]: + """Select session data; allow absence only for fixed providers without transcripts.""" home_info = role_home.lstat() if not stat.S_ISDIR(home_info.st_mode) or home_info.st_uid != os.getuid(): raise ValueError("continuation role home is not an owned directory") codex = role_home / "codex" if not codex.is_dir() or codex.is_symlink(): + if allow_absent and not os.path.lexists(codex): + return [] raise ValueError("continuation Codex home is unavailable") paths = [ "codex/" + file.name @@ -89,6 +93,8 @@ def session_state_paths(role_home: Path, session_id: str) -> list[str]: if session_id in file.name and (file.is_file() or file.is_symlink()) ) if not any(session_id in Path(path).name for path in paths): + if allow_absent: + return [] raise ValueError("continuation session transcript is missing") return sorted(paths) diff --git a/runtime/src/devflow_temporal/supervisor.py b/runtime/src/devflow_temporal/supervisor.py index d8677a8d..9bf0ad76 100644 --- a/runtime/src/devflow_temporal/supervisor.py +++ b/runtime/src/devflow_temporal/supervisor.py @@ -550,6 +550,11 @@ async def _run(self, request: dict[str, Any]) -> dict[str, Any]: from .delivery_resources import read_private, write_private output_candidate = await writer_thread(spec, self._completed_candidate, request, result) + if spec.get("provider") == "fake": + # This fixed fixture role cannot launch candidate-controlled children. + # Bind the supervisor's joined-process observation to its result. + result.setdefault("cleanup", "confirmed") + write_private(result_path, result) metadata = read_private(start_path) metadata.update(output_candidate=output_candidate, result_digest=digest(result)) write_private(start_path, metadata) diff --git a/runtime/tests/test_delivery_checkpoint_fake_temporal.py b/runtime/tests/test_delivery_checkpoint_fake_temporal.py new file mode 100644 index 00000000..c566d91b --- /dev/null +++ b/runtime/tests/test_delivery_checkpoint_fake_temporal.py @@ -0,0 +1,228 @@ +"""Public recovery through a real Temporal controller and the fixed fake provider.""" + +from __future__ import annotations + +import asyncio +import json +import shutil +from contextlib import suppress +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest +from temporal_test_server import local_temporal +from temporalio import activity +from temporalio.worker import UnsandboxedWorkflowRunner, Worker +from test_delivery_store import service as service + +from devflow_temporal import delivery_activities as activities +from devflow_temporal.delivery_api import _dispatch_start, create_app +from devflow_temporal.delivery_broker import DeliveryBroker +from devflow_temporal.delivery_config import DeliveryConfig +from devflow_temporal.delivery_continuation import session_state_paths +from devflow_temporal.delivery_resource_maintenance import prune +from devflow_temporal.delivery_store import DeliveryStore +from devflow_temporal.delivery_workflow import DeliveryWorkflow + + +@pytest.mark.asyncio +async def test_public_recovery_resumes_real_fake_implementation(service, monkeypatch): + """Catch a false resumable ACK when the real fake role cannot be continued.""" + async with local_temporal() as environment: + original, request = service + config = json.loads(original.config.path.read_text()) + config["temporal_address"] = environment.client.service_client.config.target_host + config["workspace_custody_version"] = 1 + config["repositories"]["fixture"]["allowed_paths"].append("devflow-fake-change.txt") + original.config.path.write_text(json.dumps(config)) + store = DeliveryStore(DeliveryConfig.load(original.config.path)) + # Fake admissions normally omit terminal resource cleanup. Enable the real + # finalization activity in this fixture so it covers the retention contract. + admitted_spec = store.config.admit(request) + admitted_spec["resource_cleanup_version"] = 1 + with monkeypatch.context() as admission: + admission.setattr(DeliveryConfig, "admit", lambda *_args, **_kwargs: admitted_spec) + store.submit(request) + spec = store.effective_spec(request["run_id"]) + broker = DeliveryBroker(store, spec) + calls = [] + + @activity.defn(name="delivery_precheck") + async def stopped_gate(body): + calls.append(("stopped-gate", body["iteration"])) + return {"state": "failed", "source_unchanged": True, + "candidate_id": body["candidate"]["id"], "results": [{ + "id": "fixture-prerequisite", "passed": False, + "failure_kind": "preparation", "launched": False, + "cleanup": "confirmed"}]} + + @activity.defn(name="delivery_tracker_start") + async def tracker_start(_body): + return {"state": "consistent"} + + service_context = SimpleNamespace(store=store, config=store.config) + async with Worker( + environment.client, task_queue=store.config.queue, + workflow_runner=UnsandboxedWorkflowRunner(), workflows=[DeliveryWorkflow], + activities=[activities.delivery_project, activities.delivery_prepare, + activities.delivery_role, activities.delivery_finalize_resources, + stopped_gate, tracker_start], + interceptors=[activities.DatabaseContentionInterceptor()], + ): + await _dispatch_start( + service_context, environment.client, store.pending_starts()[0], spec) + original_handle = environment.client.get_workflow_handle("delivery-" + spec["run_id"]) + try: + stopped = await asyncio.wait_for(original_handle.result(), 30) + except BaseException: + with suppress(Exception): + await original_handle.terminate("owned fixture cleanup") + raise + + assert stopped["outcome"] == "blocked" and stopped["cleanup"] == "confirmed" + assert stopped["checks"]["prepublish"]["state"] == "failed" + assert calls == [("stopped-gate", 0)] + implementation = stopped["roles"][0] + assert implementation["status"] == "pass" and implementation["cleanup"] == "confirmed" + session_id = implementation["session_id"] + assert session_id == "fake:" + spec["run_id"] + ":implement" + candidate = stopped["candidate"] + home = Path(spec["state_dir"]) / "role-homes/implement" + (home / "codex/sessions").mkdir(parents=True, mode=0o700, exist_ok=True) + assert not list((home / "codex/sessions").rglob("*")) + # The normal provider-session contract still refuses missing transcripts. + with pytest.raises(ValueError, match="continuation session transcript is missing"): + session_state_paths(home, session_id) + with store._connect() as db: + attempt = db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND role='implement'", + (spec["run_id"],)).fetchone() + assert attempt["state"] == "finished" and attempt["cleanup"] == "confirmed" + assert json.loads(attempt["result_json"])["cleanup"] == "confirmed" + assert json.loads(Path(attempt["result_path"]).read_text()) == json.loads( + attempt["result_json"]) + # The tracker boundary is a fixture; emulate its terminal claim release. + store.state.release_work(db, spec["work_id"], "external:devflow:" + spec["run_id"]) + row = dict(db.execute("SELECT * FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone()) + + # No actual GitHub operation is made by the fixture's tracker or publisher. + monkeypatch.setattr(DeliveryStore, "_ensure_no_remote_pr", lambda *_args: None) + app = create_app(store.config.path) + app.state.delivery.store = store + url = store.config.dashboard_url.rstrip("/") + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url=url, + ) as client: + headers = {"origin": url, "x-devflow-csrf": (await client.get("/api/session")).json()[ + "csrf_token"]} + path = "/api/runs/" + spec["run_id"] + "/checkpoints" + saved = await client.post(path, headers=headers, json={ + "command_id": "fake-temporal-save", "expected_revision": row["revision"]}) + assert saved.status_code == 200, saved.text + assert saved.json()["resumable"] + pruned = prune(store, spec["run_id"]) + assert any(root["path"] == str(broker.checkout) and root["state"] == "retained" + for root in pruned["roots"]) + retained = await client.get(path + "/" + saved.json()["checkpoint_id"]) + assert retained.status_code == 200 and retained.json()["resumable"] + shutil.rmtree(Path(spec["source_path"])) + shutil.rmtree(broker.checkout) + if home.exists(): + shutil.rmtree(home) + restored = await client.post( + path + "/" + saved.json()["checkpoint_id"] + "/restore", headers=headers, + json={"command_id": "fake-temporal-restore", "expected_revision": row["revision"]}) + assert restored.status_code == 200, restored.text + assert restored.json()["execution_started"] is False + assert broker.candidate() == candidate + assert (broker.checkout / "devflow-fake-change.txt").read_text() == ( + "Deterministic fake change 0\n") + command = {"continuation_kind": "stopped_delivery_resume", + "command_id": "fake-temporal-resume", + "expected_revision": stopped["revision"], + "expected_iteration": stopped["iteration"], + "expected_candidate_id": candidate["id"], + "expected_candidate_head": candidate["head"], "additional_iterations": 1} + response = await client.post("/api/runs/" + spec["run_id"] + "/continue-repair", + headers=headers, json=command) + assert response.status_code == 200, response.text + assert response.json()["run_id"] == spec["run_id"] + assert response.json()["authorized_through_iteration"] == 1 + replay = await client.post("/api/runs/" + spec["run_id"] + "/continue-repair", + headers=headers, json=command) + assert replay.status_code == 200 and replay.json() == response.json() + + with store._connect() as db: + current = db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone() + recovery = json.loads(current["recovery_json"]) + assert recovery["candidate"] == candidate and recovery["session_id"] == session_id + assert recovery["state"]["checks"] == stopped["checks"] + assert recovery["state"]["usage"] == stopped["usage"] + assert db.execute("SELECT count(*) FROM delivery_runs").fetchone()[0] == 1 + assert store.state.claim_for(db, spec["work_id"])["owner"] == ( + "external:devflow:" + spec["run_id"]) + assert not db.execute("SELECT 1 FROM delivery_effects WHERE kind='publish'").fetchone() + + @activity.defn(name="delivery_role") + async def role(body): + if body["role"] == "implement": + assert body["resume_session"] == session_id + assert body["candidate"] == candidate + calls.append(("continued-implementation", body["iteration"])) + return await activities.delivery_role(body) + return {"status": "pass", "role": body["role"], "iteration": body["iteration"], + "session_id": "fixture-" + body["role"], "candidate": body["candidate"], + "cleanup": "confirmed", "findings": []} + + @activity.defn(name="delivery_precheck") + async def passing_gate(body): + calls.append(("passing-gate", body["iteration"])) + return {"state": "passed", "candidate_id": body["candidate"]["id"]} + + @activity.defn(name="delivery_publish") + async def publish(body): + calls.append(("fixture-publication", body["iteration"])) + return {"number": 7, "head": body["candidate"]["head"], + "candidate": body["candidate"], "state": "OPEN"} + + @activity.defn(name="delivery_checks") + async def checks(body): + return {"state": "passed", "candidate_id": body["candidate"]["id"]} + + @activity.defn(name="delivery_ci") + async def ci(body): + return {"state": "passed", "head": body["pull_request"]["head"]} + + @activity.defn(name="delivery_tracker") + async def tracker(_body): + return {"state": "consistent"} + + async with Worker( + environment.client, task_queue=store.config.queue, + workflow_runner=UnsandboxedWorkflowRunner(), workflows=[DeliveryWorkflow], + activities=[activities.delivery_project, activities.delivery_repair_preflight, + activities.delivery_finalize_resources, role, passing_gate, publish, + checks, ci, tracker_start, tracker], + interceptors=[activities.DatabaseContentionInterceptor()], + ): + await _dispatch_start(service_context, environment.client, store.pending_starts()[0], + store.effective_spec(spec["run_id"])) + successor = environment.client.get_workflow_handle(response.json()["workflow_id"]) + try: + result = await asyncio.wait_for(successor.result(), 30) + except BaseException: + with suppress(Exception): + await successor.terminate("owned fixture cleanup") + raise + + assert result["outcome"] == "delivered" # Fixture publication and gates only. + assert result["roles"][0] == implementation + assert result["roles"][1]["session_id"] == session_id + assert result["usage"]["implement:0"] == stopped["usage"]["implement:0"] + assert calls == [("stopped-gate", 0), ("continued-implementation", 1), + ("passing-gate", 1), ("fixture-publication", 1)] + assert (broker.checkout / "devflow-fake-change.txt").read_text() == ( + "Deterministic fake change 1\n") From 64f0ad39313e9c9849aae8753c47dc74f53f0604 Mon Sep 17 00:00:00 2001 From: Eloi Date: Sun, 1 Nov 2026 18:27:57 +0100 Subject: [PATCH 3/4] fix(recovery): validate retained evidence on readback --- docs/durable-checkpoints.md | 5 +- .../devflow_temporal/delivery_checkpoints.py | 10 ++++ .../delivery_retained_checkpoint.py | 15 +++--- runtime/tests/test_delivery_checkpoints.py | 48 ++++++++++++++++++- 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/docs/durable-checkpoints.md b/docs/durable-checkpoints.md index 3d46578f..cefd64cb 100644 --- a/docs/durable-checkpoints.md +++ b/docs/durable-checkpoints.md @@ -97,8 +97,9 @@ Readback preserves the original `acknowledged_resumable` value as history. Its current `resumable` observation also compares the admitted authority, run and cumulative budget, issue owner, independent Git owner and candidate branch, original resource manifest, and the current writer witness. If the checkout is -still present, it compares its retained source and index under that witness; -an absent checkout can still be restored from the sealed artifacts. Changed or +still present, it compares its retained source and index under that witness. +Existing evidence and session files must match their sealed state; missing files +and an absent checkout can still be restored from the sealed artifacts. Changed or unavailable proof appears in `resume_blockers`. This is a read-only observation, not a reservation or permission to resume: restoration must reacquire custody and repeat its own admission checks. diff --git a/runtime/src/devflow_temporal/delivery_checkpoints.py b/runtime/src/devflow_temporal/delivery_checkpoints.py index caebe3b2..a19d2e59 100644 --- a/runtime/src/devflow_temporal/delivery_checkpoints.py +++ b/runtime/src/devflow_temporal/delivery_checkpoints.py @@ -1494,6 +1494,7 @@ def read_checkpoint(store, run_id, checkpoint_id): if blocker not in blockers: blockers.append(blocker) retained_source_unknown = False + retained_state_unknown = False try: registry = RunResources(spec, read_only=True) with registry.locked(read_only=True) as resources: @@ -1511,6 +1512,13 @@ def read_checkpoint(store, run_id, checkpoint_id): # The exclusive witness is observational and grants no later work. with store._checkpoint_writer_context(spec): writer_unknown = False + try: + from .delivery_retained_checkpoint import verify_retained_state + + verify_retained_state( + spec, value, _root(spec) / checkpoint_id, allow_missing=True) + except (OSError, ValueError, KeyError): + retained_state_unknown = True checkout = Path(spec["checkout"]) if os.path.lexists(checkout) and _index_baseline_supported(value): try: @@ -1536,6 +1544,8 @@ def read_checkpoint(store, run_id, checkpoint_id): blockers.append({"scope": "writer", "reason": "checkpoint_writer_custody_unknown"}) if retained_source_unknown: blockers.append({"scope": "source", "reason": "checkpoint_retained_source_unverified"}) + if retained_state_unknown: + blockers.append({"scope": "state", "reason": "checkpoint_retained_state_unverified"}) if feature_unknown: blockers.append({"scope": "feature", "reason": feature_unknown}) index_unknown = not _index_baseline_supported(value) diff --git a/runtime/src/devflow_temporal/delivery_retained_checkpoint.py b/runtime/src/devflow_temporal/delivery_retained_checkpoint.py index a8f31a91..4757edde 100644 --- a/runtime/src/devflow_temporal/delivery_retained_checkpoint.py +++ b/runtime/src/devflow_temporal/delivery_retained_checkpoint.py @@ -287,15 +287,18 @@ def _verify_recorded_source(spec, value, checkpoint_root, resources): "branch": branch, "head": head, "index_tree": value["source"]["index_tree"]} -def verify_retained_state(spec, value, checkpoint_root): - """Verify every declared evidence/session byte and authenticated absence.""" +def verify_retained_state(spec, value, checkpoint_root, *, allow_missing=False): + """Verify state targets; missing files can be restored when explicitly allowed.""" for relative, item in value["state_refs"].items(): target = _path(Path(spec["state_dir"]), relative) + try: + info = target.lstat() + except FileNotFoundError: + if item.get("kind") == "absent" or allow_missing: + continue + raise if item.get("kind") == "absent": - if os.path.lexists(target): - raise ValueError("retained checkpoint no-GO absence changed: " + relative) - continue - info = target.lstat() + raise ValueError("retained checkpoint no-GO absence changed: " + relative) if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1: raise ValueError("retained checkpoint evidence is not an owned file: " + relative) raw = target.read_bytes() diff --git a/runtime/tests/test_delivery_checkpoints.py b/runtime/tests/test_delivery_checkpoints.py index 13c7d8e9..fa12c51d 100644 --- a/runtime/tests/test_delivery_checkpoints.py +++ b/runtime/tests/test_delivery_checkpoints.py @@ -582,13 +582,59 @@ async def test_public_checkpoint_routes_require_csrf_and_return_verified_readbac response = await client.post(path, json=command, headers=headers) assert response.status_code == 200, response.text receipt = response.json() - assert (await client.get(path + "/" + receipt["checkpoint_id"])).json()["resumable"] + checkpoint_url = path + "/" + receipt["checkpoint_id"] + assert (await client.get(checkpoint_url)).json()["resumable"] + # Existing state targets must still match the acknowledged evidence. + # Missing targets remain eligible because restore can recreate them. + for target in (evidence, transcript): + original = target.read_bytes() + original_stat = target.stat() + target.write_text("changed retained fixture\n") + observed = (await client.get(checkpoint_url)).json() + assert observed["acknowledged_resumable"] and not observed["resumable"] + assert "checkpoint_retained_state_unverified" in { + blocker["reason"] for blocker in observed["resume_blockers"]} + verified = await client.post( + checkpoint_url + "/verify-retained", headers=headers, + json={"command_id": "verify-changed-" + target.name, + "expected_revision": row["revision"]}) + assert verified.status_code == 409 + assert target.read_text() == "changed retained fixture\n" + target.write_bytes(original) + os.utime(target, ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns)) + # An unobservable parent is not proof of an absent, restorable file. + evidence.parent.chmod(0) + try: + unreadable = (await client.get(checkpoint_url)).json() + assert not unreadable["resumable"] + assert "checkpoint_retained_state_unverified" in { + blocker["reason"] for blocker in unreadable["resume_blockers"]} + finally: + evidence.parent.chmod(0o700) + # A foreign target must not be followed or altered by observation. + original = evidence.read_bytes() + original_stat = evidence.stat() + other = state / "owned-sentinel.txt" + other.write_text("untouched fixture\n") + evidence.unlink() + evidence.symlink_to(other) + linked = (await client.get(checkpoint_url)).json() + assert not linked["resumable"] + assert "checkpoint_retained_state_unverified" in { + blocker["reason"] for blocker in linked["resume_blockers"]} + assert other.read_text() == "untouched fixture\n" + evidence.unlink() + evidence.write_bytes(original) + evidence.chmod(0o600) + os.utime(evidence, ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns)) + assert (await client.get(checkpoint_url)).json()["resumable"] checkpoint = state / "checkpoints" / receipt["checkpoint_id"] assert not list(checkpoint.rglob("auth.json")) shutil.rmtree(Path(spec["source_path"])) shutil.rmtree(broker.checkout) shutil.rmtree(home) evidence.unlink() + assert (await client.get(checkpoint_url)).json()["resumable"] response = await client.post( path + "/" + receipt["checkpoint_id"] + "/restore", json={"command_id": "http-restore", "expected_revision": row["revision"]}, From e5671ea1fef3ad5ae6c4bb5a97ff1c3f04f7582b Mon Sep 17 00:00:00 2001 From: Eloi Date: Sun, 1 Nov 2026 18:45:35 +0100 Subject: [PATCH 4/4] fix(recovery): bind retained state reads to opened files --- .../delivery_retained_checkpoint.py | 89 ++++++++++++++++--- 1 file changed, 77 insertions(+), 12 deletions(-) diff --git a/runtime/src/devflow_temporal/delivery_retained_checkpoint.py b/runtime/src/devflow_temporal/delivery_retained_checkpoint.py index 4757edde..c7cae1dc 100644 --- a/runtime/src/devflow_temporal/delivery_retained_checkpoint.py +++ b/runtime/src/devflow_temporal/delivery_retained_checkpoint.py @@ -287,22 +287,87 @@ def _verify_recorded_source(spec, value, checkpoint_root, resources): "branch": branch, "head": head, "index_tree": value["source"]["index_tree"]} +def _retained_state_bytes(target, expected, size): + """Return validated bytes, or a coherently observed initial absence.""" + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_NONBLOCK + with ExitStack() as stack: + parent = os.open(target.anchor, flags) + stack.callback(os.close, parent) + directories = [] + + def parents_unchanged(): + keys = ("st_dev", "st_ino", "st_mode", "st_uid") + try: + for owner, name, descriptor, initial in directories: + current = os.fstat(descriptor) + named = os.stat(name, dir_fd=owner, follow_symlinks=False) + if (any(getattr(initial, key) != getattr(current, key) for key in keys) + or any(getattr(current, key) != getattr(named, key) for key in keys)): + raise ValueError("retained checkpoint state parent changed") + except FileNotFoundError as exc: + raise ValueError("retained checkpoint state parent became unavailable") from exc + + def open_entry(name, entry_flags): + try: + return os.open(name, entry_flags, dir_fd=parent) + except FileNotFoundError: + parents_unchanged() + try: + os.stat(name, dir_fd=parent, follow_symlinks=False) + except FileNotFoundError: + parents_unchanged() + return None + raise ValueError("retained checkpoint state entry changed") from None + + for name in target.parts[1:-1]: + opened = open_entry(name, flags) + if opened is None: + return None + stack.callback(os.close, opened) + directories.append((parent, name, opened, os.fstat(opened))) + parent = opened + if expected is not None and os.fstat(parent).st_uid != os.getuid(): + raise ValueError("retained checkpoint destination parent is not owned") + opened = open_entry(target.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + if opened is None: + return None + stack.callback(os.close, opened) + before = os.fstat(opened) + if expected is None: + raise ValueError("retained checkpoint no-GO absence changed") + if (not stat.S_ISREG(before.st_mode) or before.st_uid != os.getuid() + or before.st_nlink != 1 or before.st_size != size + or stat.S_IMODE(before.st_mode) != expected["mode"] + or before.st_mtime_ns != expected["mtime_ns"]): + raise ValueError("retained checkpoint evidence metadata changed") + with os.fdopen(os.dup(opened), "rb") as stream: + raw = stream.read(size + 1) + keys = ("st_dev", "st_ino", "st_mode", "st_uid", "st_nlink", "st_size", + "st_mtime_ns", "st_ctime_ns") + after = os.fstat(opened) + try: + named = os.stat(target.name, dir_fd=parent, follow_symlinks=False) + except FileNotFoundError as exc: + raise ValueError("retained checkpoint evidence became unavailable") from exc + if (len(raw) != size + or any(getattr(before, key) != getattr(after, key) for key in keys) + or any(getattr(after, key) != getattr(named, key) for key in keys)): + raise ValueError("retained checkpoint evidence changed during observation") + parents_unchanged() + return raw + + def verify_retained_state(spec, value, checkpoint_root, *, allow_missing=False): """Verify state targets; missing files can be restored when explicitly allowed.""" for relative, item in value["state_refs"].items(): target = _path(Path(spec["state_dir"]), relative) - try: - info = target.lstat() - except FileNotFoundError: + expected = (None if item.get("kind") == "absent" else + read_private(checkpoint_root / item["artifact"], raw=True)) + raw = _retained_state_bytes(target, item if expected is not None else None, + len(expected) if expected is not None else 0) + if raw is None: if item.get("kind") == "absent" or allow_missing: continue - raise - if item.get("kind") == "absent": - raise ValueError("retained checkpoint no-GO absence changed: " + relative) - if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1: - raise ValueError("retained checkpoint evidence is not an owned file: " + relative) - raw = target.read_bytes() - if (raw != read_private(checkpoint_root / item["artifact"], raw=True) - or _hash(raw) != item["sha256"] or stat.S_IMODE(info.st_mode) != item["mode"] - or info.st_mtime_ns != item["mtime_ns"]): + raise FileNotFoundError("retained checkpoint evidence is absent: " + relative) + if raw != expected or _hash(raw) != item["sha256"]: raise ValueError("retained checkpoint original evidence changed: " + relative)