From 24bac13e6182ee341563105356f488a26f63ec53 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:01:14 +0100 Subject: [PATCH 01/24] feat(runtime): expose durable queued pause controls --- runtime/README.md | 25 +- .../src/devflow_temporal/delivery_client.py | 21 ++ .../src/devflow_temporal/delivery_control.py | 16 ++ runtime/src/devflow_temporal/delivery_mcp.py | 33 +++ .../delivery_operation_receipts.py | 33 +++ .../devflow_temporal/delivery_operations.py | 21 +- runtime/tests/test_delivery_mcp.py | 26 ++ .../test_delivery_pause_intake_public.py | 234 ++++++++++++++++++ runtime/tests/test_delivery_readonly.py | 3 + 9 files changed, 408 insertions(+), 4 deletions(-) diff --git a/runtime/README.md b/runtime/README.md index ed7d86c5..2fb78ef7 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -276,7 +276,8 @@ operation ID carrying the same domain command and request returns the original operation; a conflicting command ID is refused. Observations retain separate identities so a fresh preflight can be followed by its distinct acknowledgement. -The service dispatches persisted pending work on restart. A previously running +The service dispatches persisted pending work on restart, except pending resume +operations described below. A previously running operation is unknown once its exclusive owner is gone; it is not launched again. `POST /api/operations/{id}/reconcile` checks supported original domain receipts without creating another command. Temporal updates record the exact @@ -292,6 +293,28 @@ CLI `operation-create --request envelope.json`, `operation --id ID`, running service. The corresponding MCP tools and dashboard operation panel expose the same state; none reissues an uncertain effect under a fresh ID. +### Queued pause controls + +Pause currently applies only to explicitly admitted, untimed ordinary runs before +their workflow has started. `GET /api/runs/{id}/pause-control`, CLI +`pause-control --id RUN`, MCP `get_pause_control`, and the dashboard report the +same scope, persisted observed state and blockers. A `pause_requested` or +`pausing` state is not a completed pause. Running workflows, feature children, +native work and timed runs are not yet supported by this control. + +The CLI `pause --id RUN --request FILE` and MCP `pause_run` accept an object +with caller-chosen stable `operation_id` and `command_id` values and the +`expected_revision` from pause control. `resume` and `resume_run` additionally +require its `epoch`; resume is an explicit +instruction. Both use the durable operation journal and require an already running +service. A resume operation still pending when the service restarts is held; +the user must explicitly resubmit the same operation to dispatch it. Preserve +the complete request after a timeout, inspect the operation and reconcile an +unknown result before attempting another command. A recovered +acknowledgement records the original intent; read pause control again for current +settlement or activation. The direct API and dashboard remain available with the +same command ID, revision and epoch semantics. + Routine CLI `run --id RUN` and MCP `get_run` read the versioned persisted status projection. They do not reconstruct current execution authority. Use CLI `run-detail --id RUN` or MCP `get_run_detail` for explicit full-detail diff --git a/runtime/src/devflow_temporal/delivery_client.py b/runtime/src/devflow_temporal/delivery_client.py index 2b78f373..82e85452 100644 --- a/runtime/src/devflow_temporal/delivery_client.py +++ b/runtime/src/devflow_temporal/delivery_client.py @@ -83,6 +83,27 @@ def reconcile_operation(self, operation_id: str) -> dict: "POST", "/api/operations/" + quote(operation_id, safe="") + "/reconcile", {} ) + def pause_control(self, run_id: str) -> dict: + """Read the persisted pause scope, state and blockers without starting a service.""" + return self._request("GET", "/api/runs/" + quote(run_id, safe="") + "/pause-control") + + def _pause_operation(self, run_id: str, action: str, payload: dict[str, Any]) -> dict: + keys = {"operation_id", "command_id", "expected_revision"} + if action == "resume": + keys.add("epoch") + if not isinstance(payload, dict) or set(payload) != keys: + raise ValueError("pause operation requires the exact operation and command fields") + return self.create_operation({ + "operation_id": payload["operation_id"], "run_id": run_id, "kind": action, + "request": {key: value for key, value in payload.items() if key != "operation_id"}, + }) + + def pause(self, run_id: str, payload: dict[str, Any]) -> dict: + return self._pause_operation(run_id, "pause", payload) + + def resume(self, run_id: str, payload: dict[str, Any]) -> dict: + return self._pause_operation(run_id, "resume", payload) + def submit(self, payload: dict[str, Any]) -> dict: return self._request("POST", "/api/runs", payload) diff --git a/runtime/src/devflow_temporal/delivery_control.py b/runtime/src/devflow_temporal/delivery_control.py index afd8b431..82f1d1a8 100644 --- a/runtime/src/devflow_temporal/delivery_control.py +++ b/runtime/src/devflow_temporal/delivery_control.py @@ -899,6 +899,9 @@ async def worker( "operation", "operation-reconcile", "run-operations", + "pause-control", + "pause", + "resume", "start", "status", "stop", @@ -1002,6 +1005,19 @@ def _run(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: }[args.command](args.id) print(json.dumps(result, sort_keys=True, indent=2)) return + if args.command in {"pause-control", "pause", "resume"}: + if not args.id: + parser.error("--id is required for pause controls") + caller = read_only_client(Path(args.config).expanduser().resolve(strict=True)) + if args.command == "pause-control": + result = caller.pause_control(args.id) + else: + if args.request is None: + parser.error("--request is required for pause and resume") + payload = json.loads(args.request.read_text(encoding="utf-8")) + result = getattr(caller, args.command)(args.id, payload) + print(json.dumps(result, sort_keys=True, indent=2)) + return if args.command == "capture-retained-closures": if args.request is None: parser.error("--request is required for explicit retained-history capture") diff --git a/runtime/src/devflow_temporal/delivery_mcp.py b/runtime/src/devflow_temporal/delivery_mcp.py index 24a51e22..e85da8b2 100644 --- a/runtime/src/devflow_temporal/delivery_mcp.py +++ b/runtime/src/devflow_temporal/delivery_mcp.py @@ -136,6 +136,39 @@ def read_evidence(run_id: str, evidence_id: str) -> dict: """Read one indexed, contained evidence artifact for a run.""" return read_only_client(config_path).evidence(run_id, evidence_id) + @server.tool(annotations=read) + async def get_pause_control(run_id: str) -> dict: + """Read current pause scope, eligibility, observed state and blockers. + + Only explicitly admitted before-start runs currently support pause. + This read does not start the service or authorize a transition. + """ + return await asyncio.to_thread(read_only_client(config_path).pause_control, run_id) + + @server.tool(annotations=write) + async def pause_run(run_id: str, request_json: str) -> dict: + """Request an eligible queued pause through a durable operation. + + Supply stable operation_id and command_id values and the expected_revision + from pause control. A succeeded result acknowledges pause_requested, not + that work is already paused; a pending operation has not yet done so. Keep + the complete request stable and inspect the operation after uncertainty. + The service must already be running. + """ + return await asyncio.to_thread( + read_only_client(config_path).pause, run_id, json.loads(request_json)) + + @server.tool(annotations=write) + async def resume_run(run_id: str, request_json: str) -> dict: + """Explicitly request resume of an eligible queued pause. + + Supply stable operation_id and command_id values and the expected_revision + and epoch from pause control. A resuming acknowledgement is not completed activation. Keep + the request stable after uncertainty; the service must already run. + """ + return await asyncio.to_thread( + read_only_client(config_path).resume, run_id, json.loads(request_json)) + @server.tool(annotations=write) async def answer_decision(run_id: str, request_json: str) -> dict: """Answer a question, review a plan, or authorize the recorded feature merge.""" diff --git a/runtime/src/devflow_temporal/delivery_operation_receipts.py b/runtime/src/devflow_temporal/delivery_operation_receipts.py index 104851d5..2ad68bdf 100644 --- a/runtime/src/devflow_temporal/delivery_operation_receipts.py +++ b/runtime/src/devflow_temporal/delivery_operation_receipts.py @@ -25,6 +25,39 @@ def receipt(store, operation): command_id = payload.get("command_id") if not isinstance(command_id, str): return None + if kind in {"pause", "resume"}: + # A stored acknowledgement is historical evidence of intent, not a + # fresh observation that settlement or activation has completed. + keys = {"command_id", "expected_revision"} | ({"epoch"} if kind == "resume" else set()) + revision = payload.get("expected_revision") + epoch = payload.get("epoch") + if (set(payload) != keys or not command_id or len(command_id) > 512 + or operation.get("domain_command_id") != command_id + or operation.get("request_digest") != digest({ + "run_id": run_id, "kind": kind, "request": payload}) + or type(revision) is not int or revision < 0 + or (kind == "resume" and (type(epoch) is not int or epoch < 1))): + return None + expected = {"action": kind, "root_id": run_id, "expected_revision": revision} + if kind == "resume": + expected["epoch"] = epoch + with store._connect() as db: + row = db.execute( + "SELECT root_id,request_digest,response_json FROM delivery_pause_commands " + "WHERE command_id=?", (command_id,), + ).fetchone() + if row is None or row["root_id"] != run_id or row["request_digest"] != digest(expected): + return None + response = json.loads(row["response_json"]) + state = "pause_requested" if kind == "pause" else "resuming" + if (not isinstance(response, dict) + or set(response) != {"root_id", "state", "revision", "epoch"} + or response["root_id"] != run_id or response["state"] != state + or type(response["revision"]) is not int or response["revision"] != revision + 1 + or type(response["epoch"]) is not int or response["epoch"] < 1 + or (kind == "resume" and response["epoch"] != epoch)): + return None + return response if kind == "issue-context": if set(payload) != {"command_id"}: return None diff --git a/runtime/src/devflow_temporal/delivery_operations.py b/runtime/src/devflow_temporal/delivery_operations.py index 8bb1f83f..cadb3695 100644 --- a/runtime/src/devflow_temporal/delivery_operations.py +++ b/runtime/src/devflow_temporal/delivery_operations.py @@ -44,6 +44,8 @@ "amend-scope": ("amend_scope", "admission"), "decision": ("decision", "execution"), "cancel": ("cancel", "execution"), + "pause": ("request_pause", "execution"), + "resume": ("request_resume", "execution"), "disable-limits": ("disable_delivery_limits", "execution"), "reconcile-tracker": ("reconcile_tracker", "execution"), } @@ -65,6 +67,8 @@ def _domain_namespaces(kind): # These are the existing owners' command journals. A preflight is never # bound here, so its ID may be reused by its distinct acknowledgement. + if kind in {"pause", "resume"}: + return {"pause-commands"} if kind == "issue-context": return {"issue-context"} if kind in {"decision", "cancel", "disable-limits"}: @@ -131,6 +135,9 @@ def _row(row) -> dict: "refused": "inspect_error", "failed": "inspect_error", }[value["state"]] + if (value["kind"] == "resume" and value["state"] == "pending" + and value["progress"] == "explicit resume required after restart"): + value["next_action"] = "resubmit_same_operation" return value @@ -399,14 +406,22 @@ def dispatch(operation_id): async def resume_pending(): with service.store._connect() as db: + # A resume queued but not admitted before restart must not + # open a pause without another explicit submission of the same + # operation. Running/unknown commands still reconcile by receipt. + db.execute( + "UPDATE delivery_operations SET progress=?,updated_at=? " + "WHERE kind='resume' AND state='pending'", + ("explicit resume required after restart", now()), + ) rows = db.execute( - "SELECT operation_id,state FROM delivery_operations " + "SELECT operation_id,state,kind FROM delivery_operations " "WHERE state IN ('pending','running')" ).fetchall() for row in rows: - if row["state"] == "pending": + if row["state"] == "pending" and row["kind"] != "resume": dispatch(row["operation_id"]) - else: + elif row["state"] == "running": await asyncio.to_thread(_recover_abandoned, service.store, row["operation_id"]) service.resume_pending_operations = resume_pending diff --git a/runtime/tests/test_delivery_mcp.py b/runtime/tests/test_delivery_mcp.py index 7fbbb127..35b08103 100644 --- a/runtime/tests/test_delivery_mcp.py +++ b/runtime/tests/test_delivery_mcp.py @@ -35,6 +35,9 @@ def client(path, factory): run_operations=lambda run_id: {"operations_for": run_id}, create_operation=lambda request: {"created": request}, reconcile_operation=lambda operation_id: {"reconciled": operation_id}, + pause_control=lambda run_id: {"root_id": run_id, "scope": "before_start"}, + pause=lambda run_id, request: {"root_id": run_id, **request}, + resume=lambda run_id, request: {"root_id": run_id, **request}, ) monkeypatch.setattr("devflow_temporal.delivery_mcp.client", lambda path: client(path, "write")) @@ -43,6 +46,9 @@ def client(path, factory): async with create_connected_server_and_client_session(build_server(config)) as session: tools = {tool.name: tool for tool in (await session.list_tools()).tools} assert set(tools) == { + "get_pause_control", + "pause_run", + "resume_run", "get_service", "start_service", "submit_run", @@ -71,6 +77,7 @@ def client(path, factory): "continue_repair", } for name in ( + "get_pause_control", "get_service", "list_runs", "get_run", @@ -87,6 +94,8 @@ def client(path, factory): assert tools[name].annotations.destructiveHint is False assert tools[name].annotations.openWorldHint is False for name in ( + "pause_run", + "resume_run", "start_service", "create_operation", "reconcile_operation", @@ -162,9 +171,26 @@ def client(path, factory): ): result = await session.call_tool(name, arguments) assert not result.isError and json.loads(result.content[0].text) == expected + for name, arguments, expected in ( + ("get_pause_control", {"run_id": "same-run"}, + {"root_id": "same-run", "scope": "before_start"}), + ("pause_run", {"run_id": "same-run", "request_json": + '{"operation_id":"op-pause","command_id":"pause",' + '"expected_revision":1}'}, + {"root_id": "same-run", "operation_id": "op-pause", "command_id": "pause", + "expected_revision": 1}), + ("resume_run", {"run_id": "same-run", "request_json": + '{"operation_id":"op-resume","command_id":"resume",' + '"expected_revision":3,"epoch":1}'}, + {"root_id": "same-run", "operation_id": "op-resume", "command_id": "resume", + "expected_revision": 3, "epoch": 1}), + ): + result = await session.call_tool(name, arguments) + assert not result.isError and json.loads(result.content[0].text) == expected assert seen == [(factory, config) for factory in ( "read", "write", "write", "write", "read", "write", "read", "read", "read", "read", "read", "read", "read", + "read", "read", "read", )] diff --git a/runtime/tests/test_delivery_pause_intake_public.py b/runtime/tests/test_delivery_pause_intake_public.py index a032f4e0..a9a7d3b4 100644 --- a/runtime/tests/test_delivery_pause_intake_public.py +++ b/runtime/tests/test_delivery_pause_intake_public.py @@ -3,6 +3,7 @@ import httpx import pytest from temporalio import workflow +from test_delivery_pause_intake import unstarted as unstarted from test_delivery_store import service as service from devflow_temporal.delivery_api import create_app @@ -227,3 +228,236 @@ async def race_before_start(*args, **kwargs): "WHERE root_id='run-1'").fetchone()[0] == 1 assert (await handle.describe()).run_id == description.run_id await handle.terminate('owned fixture cleanup') + + +@pytest.mark.asyncio +async def test_public_pause_operations_reconcile_lost_acknowledgements(unstarted, monkeypatch): + """A lost response never repeats a fenced command or fabricates its outcome.""" + import asyncio + + from test_delivery_pause_intake import begin_intake_settlement, controlled_no_start + + from devflow_temporal import delivery_pause as pause + from devflow_temporal import delivery_pause_intake as intake + + _controlled_cutover(monkeypatch) + store, spec, run = unstarted + root = spec['run_id'] + app = create_app(store.config.path) + origin = app.state.delivery.config.dashboard_url + calls = [] + original_pause = pause.request_pause + + def lost_pause(*args, **kwargs): + calls.append('pause') + original_pause(*args, **kwargs) + raise OSError('fixture lost acknowledgement after commit') + + monkeypatch.setattr(pause, 'request_pause', lost_pause) + + async def operation(client, headers, envelope): + response = await client.post('/api/operations', headers=headers, json=envelope) + assert response.status_code in (200, 202), response.text + async with asyncio.timeout(10): + while True: + result = (await client.get('/api/operations/' + envelope['operation_id'])).json() + if result['state'] not in {'pending', 'running'}: + return result + await asyncio.sleep(.02) + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=origin) as client: + session = await client.get('/api/session') + headers = {'Origin': origin, 'X-Devflow-CSRF': session.json()['csrf_token']} + status = (await client.get(f'/api/runs/{root}/pause-control')).json() + envelope = {'operation_id': 'pause-op', 'run_id': root, 'kind': 'pause', + 'request': {'command_id': 'pause-cmd', 'expected_revision': status['revision']}} + lost = await operation(client, headers, envelope) + assert lost['state'] == 'unknown' + assert (await client.get(f'/api/runs/{root}/pause-control')).json()[ + 'observed'] == 'pause_requested' + # Alias resolves to the original unknown operation; changed authority is refused. + alias = await client.post('/api/operations', headers=headers, + json={**envelope, 'operation_id': 'pause-alias'}) + assert alias.json()['operation_id'] == 'pause-op' + conflict = await client.post('/api/operations', headers=headers, + json={**envelope, 'operation_id': 'pause-conflict', + 'request': {**envelope['request'], 'expected_revision': 99}}) + assert conflict.status_code == 409 + assert calls == ['pause'] + + # Recreate the API against the same owned database, then reconcile only the + # exact committed domain receipt. No dispatcher or operation is replayed. + restarted = create_app(store.config.path) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=restarted), + base_url=origin) as client: + session = await client.get('/api/session') + headers = {'Origin': origin, 'X-Devflow-CSRF': session.json()['csrf_token']} + recovered = (await client.post('/api/operations/pause-op/reconcile', + headers=headers)).json() + assert recovered['state'] == 'succeeded' + assert recovered['result'] == {'root_id': root, 'state': 'pause_requested', + 'revision': 1, 'epoch': 1} + assert calls == ['pause'] + begin_intake_settlement(store, root) + intake.seal_unstarted(store, root, 1, run['revision'], + prove_unstarted=controlled_no_start(store)) + paused = pause.publish_paused(store, root, 1) + original_resume = pause.begin_resume + + def lost_resume(*args, **kwargs): + calls.append('resume') + original_resume(*args, **kwargs) + raise OSError('fixture lost acknowledgement after commit') + + monkeypatch.setattr(pause, 'begin_resume', lost_resume) + resume = {'operation_id': 'resume-op', 'run_id': root, 'kind': 'resume', + 'request': {'command_id': 'resume-cmd', 'expected_revision': paused['revision'], + 'epoch': 1}} + # Simulate a restart after transport persistence but before admission. + # Startup must leave this resume pending until the user repeats it. + import json + + from devflow_temporal.contracts import digest + + with store._connect() as db: + db.execute("""INSERT INTO delivery_operations + (operation_id,run_id,kind,stage,request_digest,request_json,state, + created_at,updated_at,progress,domain_command_id) + VALUES (?,?,'resume','execution',?,?,'pending','2026','2026','queued',?)""", + (resume['operation_id'], root, + digest({'run_id': root, 'kind': 'resume', 'request': resume['request']}), + json.dumps(resume['request']), resume['request']['command_id'])) + await restarted.state.delivery.resume_pending_operations() + held = (await client.get('/api/operations/resume-op')).json() + assert held['state'] == 'pending' and held['next_action'] == 'resubmit_same_operation' + assert (await client.get(f'/api/runs/{root}/pause-control')).json()['observed'] == 'paused' + assert calls == ['pause'] + assert (await operation(client, headers, resume))['state'] == 'unknown' + assert (await client.get(f'/api/runs/{root}/pause-control')).json()[ + 'observed'] == 'resuming' + result = (await client.post('/api/operations/resume-op/reconcile', headers=headers)).json() + assert result['state'] == 'succeeded' + assert result['result'] == {'root_id': root, 'state': 'resuming', + 'revision': paused['revision'] + 1, 'epoch': 1} + assert calls == ['pause', 'resume'] + # A mismatched request cannot turn another command's receipt into success. + with store._connect() as db: + db.execute("UPDATE delivery_operations SET state='unknown',result_json=NULL," + "request_json=? WHERE operation_id='resume-op'", + ('{"command_id":"resume-cmd","expected_revision":999,"epoch":1}',)) + mismatch = (await client.post('/api/operations/resume-op/reconcile', + headers=headers)).json() + assert mismatch['state'] == 'unknown' + # Even another genuine receipt cannot be substituted for this operation. + with store._connect() as db: + db.execute("UPDATE delivery_operations SET kind='pause',request_json=?," + "domain_command_id='pause-cmd' WHERE operation_id='resume-op'", + ('{"command_id":"pause-cmd","expected_revision":0}',)) + substituted = (await client.post('/api/operations/resume-op/reconcile', + headers=headers)).json() + assert substituted['state'] == 'unknown' + assert calls == ['pause', 'resume'] + + +@pytest.mark.asyncio +async def test_cli_and_mcp_use_the_same_durable_pause_commands(unstarted, monkeypatch, tmp_path): + """Actual CLI and MCP clients share the public API and never start the service.""" + import asyncio + import json + import socket + import subprocess + import sys + + import uvicorn + from mcp.shared.memory import create_connected_server_and_client_session + from test_delivery_pause_intake import begin_intake_settlement, controlled_no_start + + from devflow_temporal import delivery_pause as pause + from devflow_temporal import delivery_pause_intake as intake + from devflow_temporal.delivery_mcp import build_server + + _controlled_cutover(monkeypatch) + store, spec, run = unstarted + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + listener.listen() + listener.setblocking(False) + port = listener.getsockname()[1] + host = f'http://127.0.0.1:{port}' + config = json.loads(store.config.path.read_text()) + config['dashboard_url'] = host + store.config.path.write_text(json.dumps(config)) + app = create_app(store.config.path) + + async def dormant(): + await asyncio.Event().wait() + + app.state.delivery.dispatch_loop = dormant + app.state.delivery.pause_intake_loop = dormant + server = uvicorn.Server(uvicorn.Config(app, host='127.0.0.1', port=port, log_level='error')) + serving = asyncio.create_task(server.serve(sockets=[listener])) + + async def cli(command, request=None): + argv = [sys.executable, '-m', 'devflow_temporal.delivery_control', + '--config', str(store.config.path), command, '--id', spec['run_id']] + if request is not None: + request_path = tmp_path / (command + '.json') + request_path.write_text(json.dumps(request)) + argv += ['--request', str(request_path)] + result = await asyncio.to_thread(subprocess.run, argv, capture_output=True, text=True, + timeout=15) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + async def completed(client, op): + async with asyncio.timeout(10): + while True: + result = (await client.get('/api/operations/' + op)).json() + if result['state'] not in {'pending', 'running'}: + return result + await asyncio.sleep(.02) + + try: + async with asyncio.timeout(10): + while not server.started: + await asyncio.sleep(.02) + assert (await cli('pause-control'))['scope'] == 'before_start' + request = {'operation_id': 'cross-pause', 'command_id': 'cross-pause-cmd', + 'expected_revision': 0} + first = await cli('pause', request) + assert first['operation_id'] == 'cross-pause' + async with httpx.AsyncClient(base_url=host) as client: + result = await completed(client, 'cross-pause') + assert result['state'] == 'succeeded' and result['result']['state'] == 'pause_requested' + async with create_connected_server_and_client_session( + build_server(store.config.path) + ) as mcp: + status = await mcp.call_tool('get_pause_control', {'run_id': spec['run_id']}) + assert not status.isError + assert json.loads(status.content[0].text)['observed'] == 'pause_requested' + repeated = await mcp.call_tool('pause_run', { + 'run_id': spec['run_id'], 'request_json': json.dumps(request)}) + assert not repeated.isError + assert json.loads(repeated.content[0].text)['operation_id'] == 'cross-pause' + begin_intake_settlement(store, spec['run_id']) + intake.seal_unstarted(store, spec['run_id'], 1, run['revision'], + prove_unstarted=controlled_no_start(store)) + paused = pause.publish_paused(store, spec['run_id'], 1) + resume = {'operation_id': 'cross-resume', 'command_id': 'cross-resume-cmd', + 'expected_revision': paused['revision'], 'epoch': 1} + response = await mcp.call_tool('resume_run', { + 'run_id': spec['run_id'], 'request_json': json.dumps(resume)}) + assert not response.isError + assert json.loads(response.content[0].text)['operation_id'] == 'cross-resume' + result = await completed(client, 'cross-resume') + assert result['state'] == 'succeeded' and result['result']['state'] == 'resuming' + repeated = await cli('resume', resume) + assert repeated['operation_id'] == 'cross-resume' + assert (await cli('pause-control'))['observed'] == 'resuming' + with store._connect() as db: + assert db.execute('SELECT COUNT(*) FROM delivery_pause_commands ' + 'WHERE root_id=?', (spec['run_id'],)).fetchone()[0] == 2 + finally: + server.should_exit = True + await asyncio.wait_for(serving, 10) + listener.close() diff --git a/runtime/tests/test_delivery_readonly.py b/runtime/tests/test_delivery_readonly.py index 84c27653..88380c6b 100644 --- a/runtime/tests/test_delivery_readonly.py +++ b/runtime/tests/test_delivery_readonly.py @@ -12,6 +12,7 @@ READ_TOOLS = [ ('get_service', {}), ('list_runs', {}), ('get_run', {'run_id': 'absent'}), ('get_run_detail', {'run_id': 'absent'}), + ('get_pause_control', {'run_id': 'absent'}), ('get_operation', {'operation_id': 'absent'}), ('get_run_operations', {'run_id': 'absent'}), ('read_evidence', {'run_id': 'absent', 'evidence_id': 'missing'}), @@ -68,12 +69,14 @@ def request(_caller, method, path): 'run': {'run_id': 'run/one', 'protocol_revision': 7}, } caller.detail('run/one') + caller.pause_control('run/one') caller.operation('operation/one') caller.run_operations('run/one') caller.evidence('run/one', 'artifact/one') assert calls == [ ('GET', '/api/service'), ('GET', '/api/runs'), ('GET', '/api/runs/run%2Fone/status'), ('GET', '/api/runs/run%2Fone'), + ('GET', '/api/runs/run%2Fone/pause-control'), ('GET', '/api/operations/operation%2Fone'), ('GET', '/api/runs/run%2Fone/operations'), ('GET', '/api/runs/run%2Fone/evidence/artifact%2Fone'), ] From 11a91d46978198fdb55632f2b09d1d2d5b5a59b4 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:05:52 +0100 Subject: [PATCH 02/24] fix(recovery): report lifecycle custody blockers --- docs/temporal-runtime.md | 23 +++ .../devflow_temporal/delivery_gate_retry.py | 153 ++++++++++++------ .../devflow_temporal/delivery_lifecycle.py | 67 +++++++- .../delivery_stopped_resume.py | 125 ++++++++++---- .../src/devflow_temporal/delivery_store.py | 32 +++- runtime/tests/test_delivery_gate_retry.py | 61 +++++++ .../test_delivery_store_lifecycle_240.py | 64 ++++++++ .../test_delivery_unpublished_cleanup.py | 44 ++++- .../test_delivery_unused_budget_resume.py | 22 ++- 9 files changed, 506 insertions(+), 85 deletions(-) diff --git a/docs/temporal-runtime.md b/docs/temporal-runtime.md index 242aac20..7fe0ef7c 100644 --- a/docs/temporal-runtime.md +++ b/docs/temporal-runtime.md @@ -238,6 +238,29 @@ For an ordinary published run stopped with `browser QA child cleanup is unknown` For an ordinary published run whose native roles have finished but whose resource cleanup remains unknown, `reconcile-attempt --id RUN --role cleanup --iteration N` authenticates the exact closed workflow and re-observes its native resources before closing its claim. An unpublished run is also eligible after it has an accepted plan and an exact completed implementation/source checkpoint. Its closure is recorded as `unpublished`, with the closed execution identity, original checks/results, current source and work binding, attempt/effect inventory, native observation and tracker acknowledgement. It independently observes the configured origin and absence of the owned remote branch/PR before finalization and again before claim release. Pre-intake runs, a newer issue owner, changed work bindings, uncertain effects or live native custody prevent closure. Repeating the command re-observes its exact acknowledged custody without starting another role, admitting a successor or rewriting the historical verdict. Dirty retained source, publication state, failure classification and original repair allowance remain intact. Workspace dependency cleanup uses the declaration recorded when the runtime acquired custody, so permitted package script edits do not invalidate deletion of those same owned folders; replacement folders and symlinks remain protected. +Recovery callers can inspect a specific request before admission using +`assess-transition --id RUN --request FILE`; the JSON file contains `kind` and the +corresponding recovery `request`. The API accepts the same object at +`POST /api/runs/RUN/assess-transition`. Version 1 returns a lifecycle observation, +an advisory transition for supported routes, candidate and revision observations, +and structured blockers. A blocker identifies its owner, the missing or unknown +fact, a next action, and whether new authorization or user input is required. +`user_input_required: null` means that need has not been established; it is not a +request for approval. `satisfied: null` means the fact is unknown, while `false` +means the observed predicate did not hold. Some older owner adapters still return +a general unknown when they cannot authenticate a more specific fact. + +The lifecycle vocabulary distinguishes pending, running, stopping, stopped, +recovering, terminal and unknown projections. A projection or successful +assessment is not a custody grant: the owning admission rechecks the exact +candidate, original authority, effects, cleanup and finite budget under its +existing fences. The transition table does not choose a route. Pause states are +reserved for the pause owner; queued pause support does not prove a running +workflow has settled. An ordinary preparation failure before plan acceptance +cannot use implementation resume without an accepted plan. Inspect the retained +intake and original custody; do not treat a missing plan or unknown cleanup as a +completed implementation or a released owner. + After that closure, use the public `repair-admission-preflight --id RUN --request FILE` and `continue-repair --id RUN --request FILE` route with `continuation_kind: "stopped_delivery_resume"`, a stable unique `command_id`, the closed protocol `expected_revision`, `expected_iteration`, and current `expected_candidate_id`/`expected_candidate_head`. A passed implementation stopped with exact unknown prepublication cleanup resumes at the sealed iteration's checks with `additional_iterations: 0`. Its exact durable implementation result and unchanged candidate remain mandatory; all publication, independent review, QA, verification, required CI and tracker gates still run. A provider monitor lost after launch remains an unknown verdict. For the narrow original unpublished implementation checkpoint, public cleanup can independently authenticate finished teardown without completing monitoring or inventing a provider session. The closed role, durable supervisor result, exact private request/journal hashes, input candidate, current partial output, provider identity and unknown outcome are bound separately. A later explicit `stopped_delivery_resume` must grant one or two unused original iterations (`additional_iterations: 1` or `2`), starts at the next iteration in a fresh implementation home/session, and receives no invented model finding from the old infrastructure failure. The old result, role and monitoring state remain unchanged. Subsequent normal resumes retain only those exact historical teardown receipts through the immutable admission chain; a different unknown role, changed output/evidence or current live custody still blocks them. Unknown external effects are never replayed. diff --git a/runtime/src/devflow_temporal/delivery_gate_retry.py b/runtime/src/devflow_temporal/delivery_gate_retry.py index a68a766e..96c1ecf2 100644 --- a/runtime/src/devflow_temporal/delivery_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_gate_retry.py @@ -766,46 +766,85 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= raise ValueError('gate retry requires the existing trusted native execution policy') unpublished_error = (state.get('error') if prepublication_preparation_failed(state) else 'prepublication repair limit exhausted') - if ((kind == PRELAUNCH_KIND and not prelaunch) - or (previous is not None and not ( - renewed or published_after_recovery or prelaunch - or report_retry or ci_only or transient_retry or attribution_retry - or verification_retry or browser_artifact_retry or prerequisite_retry)) - or row['phase'] != 'blocked' or row['outcome'] != 'blocked' - or row['execution_state'] != 'blocked' - or row['cleanup'] != ('unknown' if prelaunch else 'confirmed') - or state.get('phase') != 'blocked' or state.get('outcome') != 'blocked' - or (prelaunch and state.get('cleanup') != 'unknown') - or (not prelaunch and state.get('cleanup') != 'confirmed' and reconciled is None) - or row['error'] != state.get('error') - or (not (transient_retry or transient_ci or masked_exhaustion) - and state.get('error') not in ({'repair limit exhausted', - 'required CI did not confirm this PR head'} - if report_retry - else {'required CI did not confirm this PR head' - if ci_only - else 'browser QA child cleanup is unknown' - if browser_prelaunch else - 'local check process cleanup is unknown' if prelaunch - else unpublished_error - if unpublished else state.get('error') - if (controller_only or attribution_retry - or browser_artifact_retry or prerequisite_retry) - else 'repair limit exhausted'})) - or row['protocol_revision'] != state.get('revision') - or row['iteration'] != state.get('iteration') - or closed['request_digest'] != row['request_digest'] - or closed['recovery_digest'] != (digest(previous) if previous else None) - or not implementation or (implementation.get('status') != 'pass' - and not (controller_only or attribution_retry - or browser_artifact_retry - or layout_report or verification_retry)) - or not failed_gate - or any(a['state'] != 'finished' or a['cleanup'] != 'confirmed' for a in attempts) - or any((e['state'] != 'complete' or not e['observed_json']) - and e != (browser_observation or {}).get('pending_effect') for e in effects) - or (claim is not None and not prelaunch)): - raise ValueError('gate retry requires a closed, finalized failed gate and released claim') + # Report the original guard's independent facts through the shared + # evaluator. The same snapshot is used for observation and + # admission, so neither surface needs to interpret English error strings. + from .delivery_lifecycle import lifecycle_from_row, require_observed_predicates + + checkpoint_matches = not ( + (kind == PRELAUNCH_KIND and not prelaunch) + or (previous is not None and not ( + renewed or published_after_recovery or prelaunch or report_retry or ci_only + or transient_retry or attribution_retry or verification_retry + or browser_artifact_retry or prerequisite_retry)) + or row['phase'] != 'blocked' or row['outcome'] != 'blocked' + or row['execution_state'] != 'blocked' + or state.get('phase') != 'blocked' or state.get('outcome') != 'blocked' + or row['error'] != state.get('error') + or row['protocol_revision'] != state.get('revision') + or row['iteration'] != state.get('iteration') + or closed['request_digest'] != row['request_digest'] + or closed['recovery_digest'] != (digest(previous) if previous else None)) + cleanup_matches = (row['cleanup'] == ('unknown' if prelaunch else 'confirmed') + and ((state.get('cleanup') == 'unknown') if prelaunch else + (state.get('cleanup') == 'confirmed' or reconciled is not None))) + verdict_matches = ( + (transient_retry or transient_ci or masked_exhaustion + or state.get('error') in ({'repair limit exhausted', + 'required CI did not confirm this PR head'} if report_retry + else {'required CI did not confirm this PR head' if ci_only + else 'browser QA child cleanup is unknown' if browser_prelaunch + else 'local check process cleanup is unknown' if prelaunch + else unpublished_error if unpublished + else state.get('error') if (controller_only or attribution_retry + or browser_artifact_retry or prerequisite_retry) + else 'repair limit exhausted'})) + and bool(implementation) + and (implementation.get('status') == 'pass' or controller_only or attribution_retry + or browser_artifact_retry or layout_report or verification_retry) + and failed_gate) + frozen_candidate = json.loads(row['candidate_json'] or 'null') + candidate_matches = (isinstance(frozen_candidate, dict) + and not any(candidate.get(k) != frozen_candidate.get(k) for k in candidate) + and frozen_candidate == state.get('candidate')) + attempts_settled = all(a['state'] == 'finished' and a['cleanup'] == 'confirmed' + for a in attempts) + effects_settled = all((e['state'] == 'complete' and bool(e['observed_json'])) + or e == (browser_observation or {}).get('pending_effect') + for e in effects) + released = claim is None or prelaunch + require_observed_predicates(kind, 'delivery_gate_retry', {'row': row, 'closed': closed}, [ + {'code': 'gate_checkpoint_unproven', 'satisfied': checkpoint_matches, + 'missing_or_unknown_fact': 'matching stopped run and closed predecessor', + 'next_action': 'inspect the closed run and its retained recovery history', + 'user_input_required': None}, + {'code': 'gate_cleanup_unconfirmed', 'satisfied': True if cleanup_matches else None, + 'missing_or_unknown_fact': 'confirmed cleanup or authenticated later reconciliation', + 'next_action': 'reconcile the original cleanup and custody', + 'user_input_required': None}, + {'code': 'gate_candidate_changed', 'satisfied': candidate_matches, + 'missing_or_unknown_fact': 'unchanged candidate matching the closed checkpoint', + 'next_action': 'inspect the retained candidate and obtain fresh owner proof', + 'user_input_required': None}, + {'code': 'failed_gate_unproven', 'satisfied': verdict_matches, + 'missing_or_unknown_fact': 'failed gate and its original implementation verdict', + 'next_action': 'inspect the retained gate and implementation evidence', + 'user_input_required': None}, + {'code': 'gate_attempts_unsettled', 'satisfied': True if attempts_settled else None, + 'missing_or_unknown_fact': 'finished attempts and confirmed cleanup', + 'next_action': 'reconcile the original attempt custody', + 'user_input_required': None}, + {'code': 'gate_effects_unsettled', 'satisfied': True if effects_settled else None, + 'missing_or_unknown_fact': 'settled effects with observed outcomes', + 'next_action': 'reconcile the original effects', 'user_input_required': None}, + {'code': 'gate_owner_conflict', 'satisfied': released, + 'missing_or_unknown_fact': 'released work claim or allowed prelaunch custody', + 'next_action': 'inspect and reconcile the current work owner', + 'user_input_required': None}, + ], lifecycle=lifecycle_from_row(row), owner_proof=( + True if (checkpoint_matches and cleanup_matches and candidate_matches + and attempts_settled and effects_settled and released) else None), + legacy_message='gate retry requires a closed, finalized failed gate and released claim') frozen = json.loads(row['candidate_json']) publication = json.loads(row['pr_json'] or 'null') remote = _git(broker.source, 'ls-remote', 'origin', 'refs/heads/' + spec['branch']) @@ -822,14 +861,32 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= and remote and remote.split()[0] == candidate['head'] and not _git(broker.checkout, 'status', '--porcelain', '--untracked-files=no') ) - if (any(candidate.get(k) != frozen.get(k) for k in candidate) - or frozen != state.get('candidate') or not publication_matches - or _git(broker.checkout, 'branch', '--show-current') - != (spec.get('local_branch', spec['branch']) if unpublished else spec['branch']) - or _git(broker.checkout, 'remote', 'get-url', '--push', 'origin') != spec['origin_url'] - or _git(broker.source, 'remote', 'get-url', 'origin') != spec['origin_url'] - or outside_scope(spec['policy'], broker._changed_paths(), checkout=broker.checkout)): - raise ValueError('gate retry lost unchanged owned source or exact published PR head') + candidate_unchanged = (not any(candidate.get(k) != frozen.get(k) for k in candidate) + and frozen == state.get('candidate')) + repository_unchanged = ( + _git(broker.checkout, 'branch', '--show-current') + == (spec.get('local_branch', spec['branch']) if unpublished else spec['branch']) + and _git(broker.checkout, 'remote', 'get-url', '--push', 'origin') == spec['origin_url'] + and _git(broker.source, 'remote', 'get-url', 'origin') == spec['origin_url'] + and not outside_scope(spec['policy'], broker._changed_paths(), checkout=broker.checkout)) + require_observed_predicates(kind, 'delivery_gate_retry', { + 'row': row, 'closed': closed, 'candidate': candidate, 'publication': publication, + }, [ + {'code': 'gate_candidate_changed', 'satisfied': candidate_unchanged, + 'missing_or_unknown_fact': 'unchanged candidate matching the closed checkpoint', + 'next_action': 'inspect the retained candidate and obtain fresh owner proof', + 'user_input_required': None}, + {'code': 'gate_publication_unconfirmed', 'satisfied': bool(publication_matches), + 'missing_or_unknown_fact': 'exact unpublished or published branch and PR identity', + 'next_action': 'reconcile the original publication and branch', + 'user_input_required': None}, + {'code': 'gate_repository_authority_changed', 'satisfied': repository_unchanged, + 'missing_or_unknown_fact': 'original branch, destination and authorized source scope', + 'next_action': 'inspect the retained repository authority and changed paths', + 'user_input_required': None}, + ], lifecycle=lifecycle_from_row(row), owner_proof=( + True if candidate_unchanged and publication_matches and repository_unchanged else None), + legacy_message='gate retry lost unchanged owned source or exact published PR head') load_execution_config(spec) with store._connect() as db: binding = work_binding(store, spec, db) diff --git a/runtime/src/devflow_temporal/delivery_lifecycle.py b/runtime/src/devflow_temporal/delivery_lifecycle.py index a9278a5b..bab60eb9 100644 --- a/runtime/src/devflow_temporal/delivery_lifecycle.py +++ b/runtime/src/devflow_temporal/delivery_lifecycle.py @@ -12,6 +12,45 @@ VERSION = 1 +# These are observations, not ownership or execution grants. Pause is reserved +# for the pause owner; an unknown state never establishes safe custody. +LIFECYCLE_STATES = frozenset({ + 'pending', 'running', 'stopping', 'stopped', 'recovering', 'terminal', 'unknown', +}) +PAUSE_EXTENSION_STATES = frozenset({'pausing', 'paused', 'resuming'}) + +# Route identities are explicit. Each owner authenticates the full source +# checkpoint and rechecks it when acquiring admission custody. The table does +# not infer a route from an error or authorize a transition by itself. +TRANSITION_TABLE = { + kind: {'from': ['stopped'], 'to': 'recovering'} for kind in ( + 'repair_continuation', 'stopped_delivery_resume', 'published_gate_retry', + 'prepublication_gate_retry', 'published_check_prelaunch_retry', + 'published_ci_retry', 'published_controller_retry', 'recover_publication', + 'retry_prelaunch', 'stopped_resource_closure', 'abandon_pending_resource_closure', + 'published_head_validation', 'investigation_gates_only', 'amend_scope', + ) +} +# Cancelled feature workers and revision-owned completed workers can be resumed +# only when their owner supplies the additional historical authority. +TRANSITION_TABLE['stopped_delivery_resume']['from'].append('terminal') + + +def lifecycle_from_row(row: Mapping[str, Any]) -> str: + """Describe a persisted projection without treating it as a custody proof.""" + phase, execution, outcome = (row.get(key) for key in ('phase', 'execution_state', 'outcome')) + if execution == 'blocked' and phase == outcome == 'blocked': + return 'stopped' + if execution == 'terminal' and outcome in {'delivered', 'cancelled', 'failed'}: + return 'terminal' + if execution == 'stopping': + return 'stopping' + if execution in {'queued', 'pending', 'running'}: + if row.get('recovery_json'): + return 'recovering' + return 'running' if execution == 'running' else 'pending' + return 'unknown' + # Public command names only; this table never selects a recovery route or grants it. _NEXT_ACTIONS = { 'repair_continuation': 'continue-repair', @@ -61,7 +100,7 @@ def evaluate_transition(facts: Mapping[str, Any], request: Mapping[str, Any]) -> def block(code, fact, action, authorization=False, satisfied=None): blockers.append({'code': code, 'owner': owner, 'missing_or_unknown_fact': fact, 'next_action': action, 'new_authorization_required': authorization, - 'satisfied': satisfied}) + 'satisfied': satisfied, 'user_input_required': None}) if facts.get('owner_proof') is not True: block('owner_proof_unknown', 'authenticated owner and custody proof', @@ -97,11 +136,13 @@ def block(code, fact, action, authorization=False, satisfied=None): 'next_action': predicate['next_action'], 'new_authorization_required': predicate.get('new_authorization_required', False), 'satisfied': predicate.get('satisfied'), + 'user_input_required': predicate.get('user_input_required'), }) observed = {key: facts.get(key) for key in ( 'revision', 'iteration', 'candidate_id', 'candidate_head', 'publication_number')} return {'version': VERSION, 'operation': operation, 'normalized_lifecycle': facts.get('normalized_lifecycle', 'unknown'), + 'transition': TRANSITION_TABLE.get(operation), 'eligible': not blockers, 'observed_revision': observed['revision'], 'candidate': {'id': observed['candidate_id'], 'head': observed['candidate_head']}, 'publication': ({'number': observed['publication_number']} @@ -121,6 +162,30 @@ def require_transition(facts: Mapping[str, Any], request: Mapping[str, Any]) -> return assessment +def require_observed_predicates(operation: str, owner: str, proof: Any, + predicates: list[dict[str, Any]], *, + lifecycle: str = 'unknown', owner_proof: bool | None = True, + legacy_message: str | None = None + ) -> dict[str, Any]: + """Refuse exact observed predicates in both preflight and owner admission. + + This helper never substitutes a digest for custody: callers must establish + the authenticated observation and all owner-specific predicates themselves. + """ + from .contracts import digest + + try: + return require_transition({ + 'operation': operation, 'owner': owner, 'owner_proof': owner_proof, + 'normalized_lifecycle': lifecycle, 'proof_digest': digest(proof), + 'comparisons': [], 'predicates': predicates, + }, {}) + except RecoveryRefused as exc: + if legacy_message is not None: + exc.args = (legacy_message,) + raise + + def require_same_proof(operation: str, owner: str, observed_digest: str, expected_digest: str) -> dict[str, Any]: """Recheck the owner's full seal under its admission custody.""" diff --git a/runtime/src/devflow_temporal/delivery_stopped_resume.py b/runtime/src/devflow_temporal/delivery_stopped_resume.py index 2b64c72c..6557c980 100644 --- a/runtime/src/devflow_temporal/delivery_stopped_resume.py +++ b/runtime/src/devflow_temporal/delivery_stopped_resume.py @@ -201,7 +201,14 @@ def worker_allowance(spec, iteration, iterations, *, credit=None): owner = shared.token(current) shared.repair_allowance(owner, reference, current=True) if iterations > budget['maximum'] - budget['used'] + int(bool(credit)): - raise ValueError('feature product repair limit exhausted') + from .delivery_lifecycle import require_observed_predicates + + require_observed_predicates(KIND, 'feature_registry', {'owner': owner, 'budget': budget}, [{ + 'code': 'product_budget_exhausted', 'satisfied': False, + 'missing_or_unknown_fact': 'remaining authenticated feature product repair allowance', + 'next_action': 'inspect the original feature repair allowance', + 'new_authorization_required': True, 'user_input_required': None, + }], lifecycle='unknown', legacy_message='feature product repair limit exhausted') return {'run_id': spec['run_id'], 'owner': owner, 'receipt': reference, 'start_iteration': iteration, 'maximum_iteration': iteration + iterations, **({'credit': credit} if credit else {})} @@ -1108,25 +1115,64 @@ def _snapshot(store, run_id, *, _revision=None, review_subject=False, _observati raise ValueError('stopped worker cleanup belongs to a different closed execution') cleanup_authority = authenticated state_checks, row_checks = closed_checks(row, state, cleanup_authority) - if (row['phase'] not in stopped or row['outcome'] != row['phase'] - or row['execution_state'] != expected_execution or row['cleanup'] != 'confirmed' - or state.get('run_id') != run_id or state.get('phase') != row['phase'] - or state.get('outcome') != row['outcome'] - or (state.get('cleanup') != 'confirmed' and cleanup_authority is None) - or state.get('execution_state') != expected_execution - or closed.get('workflow_id') != predecessor_id - or state.get('candidate') != json.loads(row['candidate_json'] or 'null') - or state_checks != row_checks - or state.get('pull_request') != json.loads(row['pr_json'] or 'null') - or state.get('revision') != row['protocol_revision'] - or state.get('iteration') != row['iteration'] or state.get('error') != row['error'] - or closed['request_digest'] != row['request_digest'] - or closed['recovery_digest'] != (digest(previous) if previous else None) - or claim is not None - or any(e['state'] != 'complete' or e['observed_json'] is None for e in effects)): - raise ValueError('resume requires a closed finalized delivery with released ownership') - if not isinstance(spec.get('accepted_plan'), str) or not spec['accepted_plan'].strip(): - raise ValueError('implementation resume requires an accepted plan') + # Keep the closed verdict immutable while reporting later custody facts + # separately. These are exactly the facts checked by the original guard. + from .delivery_lifecycle import lifecycle_from_row, require_observed_predicates + + status_matches = (row['phase'] in stopped and row['outcome'] == row['phase'] + and row['execution_state'] == expected_execution and state.get('run_id') == run_id + and state.get('phase') == row['phase'] and state.get('outcome') == row['outcome'] + and state.get('execution_state') == expected_execution) + cleanup_matches = (row['cleanup'] == 'confirmed' + and (state.get('cleanup') == 'confirmed' or cleanup_authority is not None)) + identity_matches = (closed.get('workflow_id') == predecessor_id + and closed['request_digest'] == row['request_digest'] + and closed['recovery_digest'] == (digest(previous) if previous else None)) + checkpoint_matches = (state.get('candidate') == json.loads(row['candidate_json'] or 'null') + and state_checks == row_checks + and state.get('pull_request') == json.loads(row['pr_json'] or 'null') + and state.get('revision') == row['protocol_revision'] + and state.get('iteration') == row['iteration'] and state.get('error') == row['error']) + plan_available = (isinstance(spec.get('accepted_plan'), str) + and bool(spec['accepted_plan'].strip())) + effects_settled = all(e['state'] == 'complete' and e['observed_json'] is not None + for e in effects) + require_observed_predicates(KIND, 'delivery_stopped_resume', {'row': row, 'closed': closed}, [ + {'code': 'stopped_state_unproven', 'satisfied': status_matches, + 'missing_or_unknown_fact': 'matching stopped or supported terminal run state', + 'next_action': 'inspect the retained run and closed predecessor', + 'user_input_required': None}, + {'code': 'stopped_cleanup_unconfirmed', 'satisfied': True if cleanup_matches else None, + 'missing_or_unknown_fact': 'confirmed cleanup or authenticated later reconciliation', + 'next_action': 'reconcile the original cleanup and custody', + 'user_input_required': None}, + {'code': 'stopped_predecessor_unproven', 'satisfied': identity_matches, + 'missing_or_unknown_fact': 'matching workflow, admission and recovery identities', + 'next_action': 'inspect the closed predecessor and its original admission', + 'user_input_required': None}, + {'code': 'stopped_checkpoint_changed', 'satisfied': checkpoint_matches, + 'missing_or_unknown_fact': 'unchanged candidate, checks, publication and verdict', + 'next_action': 'inspect retained checkpoint and immutable history', + 'user_input_required': None}, + {'code': 'stopped_owner_conflict', 'satisfied': claim is None, + 'missing_or_unknown_fact': 'released original work claim', + 'next_action': 'inspect and reconcile the current work owner', + 'user_input_required': None}, + {'code': 'stopped_effects_unsettled', 'satisfied': True if effects_settled else None, + 'missing_or_unknown_fact': 'settled effects with observed outcomes', + 'next_action': 'reconcile the original effects', 'user_input_required': None}, + {'code': 'accepted_plan_unavailable', 'satisfied': plan_available, + 'missing_or_unknown_fact': 'original accepted plan for implementation resume', + 'next_action': 'inspect retained intake and preparation; no implementation resume ' + 'is authorized without an accepted plan', + 'user_input_required': None}, + ], lifecycle=lifecycle_from_row(row), owner_proof=( + True if (status_matches and cleanup_matches and identity_matches and checkpoint_matches + and claim is None and effects_settled) else None), + legacy_message=('resume requires a closed finalized delivery with released ownership' + if not (status_matches and cleanup_matches and identity_matches + and checkpoint_matches and claim is None and effects_settled) + else 'implementation resume requires an accepted plan')) if spec.get('baseline_checks_version') in (1, 2): baseline = state.get('checks', {}).get('baseline', {}) required = {check['id'] for check in spec['policy']['baseline_checks']} @@ -1258,11 +1304,11 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): with store._connect() as db: pending = None if gate_only else pending_repair(seal, db=db) ci_findings = None if gate_only else sealed_ci_findings(seal) - from .delivery_lifecycle import require_same_proof, require_transition + from .delivery_lifecycle import lifecycle_from_row, require_same_proof, require_transition lifecycle_assessment = require_transition({ 'operation': KIND, 'owner': 'delivery_stopped_resume', 'owner_proof': True, - 'normalized_lifecycle': 'stopped', 'proof_digest': digest(seal), + 'normalized_lifecycle': lifecycle_from_row(seal['row']), 'proof_digest': digest(seal), 'revision': seal['state']['revision'], 'iteration': seal['state']['iteration'], 'candidate_id': seal['candidate']['id'], 'candidate_head': seal['candidate']['head'], @@ -1271,15 +1317,28 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): interrupted = None if _revision is not None or pending else interrupted_resume(seal) if interrupted and interrupted['additional_iterations'] != command['additional_iterations']: interrupted = None - if command['additional_iterations'] == 0 and not (pending or gate_only or interrupted): - raise ValueError('zero additional iterations require an authenticated pending repair ' - 'or an interrupted iteration') - if seal.get('quarantined_implementation') and command['additional_iterations'] == 0: - raise ValueError('unknown implementation requires a separately authorized new iteration') maximum = continuation_maximum(seal['state']['iteration'], command['additional_iterations'], pending) - if maximum is None: - raise ValueError('unstarted preflight retains only its existing finite ceiling') + from .delivery_lifecycle import require_observed_predicates + + require_observed_predicates(KIND, 'delivery_stopped_resume', seal, [ + {'code': 'pending_repair_unproven', + 'satisfied': (command['additional_iterations'] != 0 + or bool(pending or gate_only or interrupted)), + 'missing_or_unknown_fact': 'authenticated pending repair or interrupted iteration', + 'next_action': 'inspect the retained repair admission and original iteration', + 'user_input_required': None}, + {'code': 'unknown_implementation_requires_new_turn', + 'satisfied': (not seal.get('quarantined_implementation') + or command['additional_iterations'] != 0), + 'missing_or_unknown_fact': 'separately authorized new implementation iteration', + 'next_action': 'review the unknown implementation and its original allowance', + 'new_authorization_required': True, 'user_input_required': True}, + {'code': 'pending_repair_ceiling_changed', 'satisfied': maximum is not None, + 'missing_or_unknown_fact': 'unchanged finite pending preflight ceiling', + 'next_action': 'inspect the retained preflight admission and its ceiling', + 'user_input_required': None}, + ], lifecycle=lifecycle_from_row(seal['row'])) credit = None if _revision is not None: from .delivery_feature_execution import registry @@ -1299,7 +1358,13 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): and not fixed_budget_allows(budget_spec, seal['state']['iteration'], command['additional_iterations'], pending=bool(pending or gate_only or interrupted))): - raise ValueError('a fixed repair budget cannot receive additional iterations') + require_observed_predicates(KIND, 'delivery_stopped_resume', seal, [{ + 'code': 'product_budget_exhausted', 'satisfied': False, + 'missing_or_unknown_fact': 'remaining original finite product repair allowance', + 'next_action': 'inspect the original repair budget and retained admission', + 'new_authorization_required': True, 'user_input_required': None, + }], lifecycle=lifecycle_from_row(seal['row']), + legacy_message='a fixed repair budget cannot receive additional iterations') amended = scope_spec(store, seal, command) browser_amendment = None if BROWSER_FIELDS <= command.keys(): diff --git a/runtime/src/devflow_temporal/delivery_store.py b/runtime/src/devflow_temporal/delivery_store.py index 6bbf5bb0..e2f11692 100644 --- a/runtime/src/devflow_temporal/delivery_store.py +++ b/runtime/src/devflow_temporal/delivery_store.py @@ -1855,7 +1855,37 @@ def continue_repair( or not fixed_budget_allows(original_budget, supplied.get('expected_iteration'), supplied['additional_iterations'])): - raise ValueError("a fixed repair budget cannot receive additional iterations") + from .delivery_lifecycle import require_observed_predicates + + requested_kind = supplied.get('continuation_kind') + with self._connect() as db: + observed = db.execute( + 'SELECT iteration,protocol_revision FROM delivery_runs WHERE run_id=?', + (run_id,)).fetchone() + iteration_matches = (observed is not None + and type(supplied.get('expected_iteration')) is int + and observed['iteration'] == supplied['expected_iteration']) + code = ('additional_iterations_not_supported' if requested_kind != KIND else + 'product_budget_exhausted' if iteration_matches else + 'budget_checkpoint_unknown') + if requested_kind != KIND: + fact = 'a recovery route that supports product iterations' + elif iteration_matches: + fact = 'remaining original finite product repair allowance' + else: + fact = 'current iteration bound to the requested budget checkpoint' + require_observed_predicates( + requested_kind or 'repair_continuation', 'delivery_budget', + {'original_authority': original_budget, 'request': supplied, + 'observed_iteration': observed['iteration'] if observed else None}, [{ + 'code': code, + 'satisfied': False if requested_kind != KIND or iteration_matches else None, + 'missing_or_unknown_fact': fact, + 'next_action': 'inspect the original budget and recovery checkpoint', + 'new_authorization_required': code == 'product_budget_exhausted', + 'user_input_required': None, + }], lifecycle='unknown', + legacy_message='a fixed repair budget cannot receive additional iterations') if (isinstance(supplied, dict) and supplied.get('continuation_kind') == 'stopped_delivery_resume'): from .delivery_stopped_resume import admit diff --git a/runtime/tests/test_delivery_gate_retry.py b/runtime/tests/test_delivery_gate_retry.py index cafceb31..3efc2d41 100644 --- a/runtime/tests/test_delivery_gate_retry.py +++ b/runtime/tests/test_delivery_gate_retry.py @@ -246,3 +246,64 @@ def missing_report(stopped, monkeypatch): closed.update(workflow_id=workflow.fetchone()[0], recovery_digest=digest(previous)) return store, broker, state, closed, {**request, 'command_id': 'missing-report-reassessment', 'expected_revision': 23}, previous, reference + + +@pytest.mark.asyncio +async def test_public_gate_retry_rechecks_candidate_and_owner(unpublished, monkeypatch): + """A valid preview cannot authorize changed source or a competing owner.""" + import copy + + import httpx + + from devflow_temporal import delivery_gate_retry + from devflow_temporal.contracts import digest + from devflow_temporal.delivery_api import create_app + + store, broker, state, command = unpublished + def renew(spec, *_args): + renewed = copy.deepcopy(spec) + renewed['policy']['measured_runtime'] = 'renewed' + renewed['policy_digest'] = digest(renewed['policy']) + return renewed + monkeypatch.setattr(delivery_gate_retry, 'prepare_runtime', renew) + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + headers = {'Origin': host, 'X-Devflow-CSRF': token} + async def assess(): + return await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': PREPUBLICATION_KIND, 'request': command}) + assert (await assess()).json()['eligible'] + path = broker.checkout / 'README.md' + original = path.read_bytes() + path.write_text('Changed after preflight\n') + changed = await assess() + response = await client.post('/api/runs/run-1/continue-repair', json=command, + headers=headers) + assert not changed.json()['eligible'] + assert 'gate_candidate_changed' in {b['code'] for b in changed.json()['blockers']} + assert response.status_code == 409 and response.json()['detail'] == changed.json() + path.write_bytes(original) + with store._connect() as db: + store.state.claim_work(db, broker.spec['work_id'], 'external:other', 'fixture') + owned = await assess() + response = await client.post('/api/runs/run-1/continue-repair', json=command, + headers=headers) + assert not owned.json()['eligible'] + assert 'gate_owner_conflict' in {b['code'] for b in owned.json()['blockers']} + assert response.status_code == 409 and response.json()['detail'] == owned.json() + with store._connect() as db: + assert not db.execute('SELECT 1 FROM delivery_gate_admissions').fetchone() + store.state.release_work(db, broker.spec['work_id'], 'external:other') + assert (await assess()).json()['eligible'] + accepted = await client.post('/api/runs/run-1/continue-repair', json=command, + headers=headers) + assert accepted.status_code == 200, accepted.text + replay = await client.post('/api/runs/run-1/continue-repair', json=command, + headers=headers) + assert replay.json() == accepted.json() + with store._connect() as db: + assert db.execute('SELECT COUNT(*) FROM delivery_gate_admissions').fetchone()[0] == 1 + assert state['error'] == 'prepublication repair limit exhausted' diff --git a/runtime/tests/test_delivery_store_lifecycle_240.py b/runtime/tests/test_delivery_store_lifecycle_240.py index 3bc62e63..73148a11 100644 --- a/runtime/tests/test_delivery_store_lifecycle_240.py +++ b/runtime/tests/test_delivery_store_lifecycle_240.py @@ -58,3 +58,67 @@ async def test_default_repair_public_assessment_and_admission_agree(service, mon assert preview.json()['blockers'][0]['code'] == 'revision_changed' accepted = await client.post(base + 'continue-repair', json=command, headers=headers) assert accepted.status_code == 200 and accepted.json()['authorized_through_iteration'] == 4 + + +@pytest.mark.asyncio +async def test_preplan_stop_refuses_implementation_resume_across_restart(service, monkeypatch): + """A closed pre-intake preparation failure must not be resumed as implementation.""" + import copy + import json + + import httpx + + from devflow_temporal.delivery_api import create_app + from devflow_temporal.delivery_config import DeliveryConfig + from devflow_temporal.delivery_store import DeliveryStore + + initial, request = service + raw = copy.deepcopy(initial.config.raw) + raw['roles']['intake'] = {'model': 'fixture', 'effort': 'low'} + initial.config.path.write_text(json.dumps(raw)) + store = DeliveryStore(DeliveryConfig.load(initial.config.path)) + request = {key: value for key, value in request.items() if key != 'accepted_plan'} + store.submit(request) + store.mark_start('run-1', accepted=True) + error = 'fixture preparation failed before planning' + store.project('run-1', phase='blocked', execution_state='blocked', event_type='blocked', + message=error, outcome='blocked', cleanup='confirmed', error=error, + protocol_revision=3, iteration=0) + with store._connect() as db: + db.execute("UPDATE delivery_runs SET workflow_id='delivery-run-1' WHERE run_id='run-1'") + store.state.release_work(db, request['work_id'], 'external:devflow:run-1') + row = db.execute('SELECT * FROM delivery_runs WHERE run_id=?', ('run-1',)).fetchone() + saved_request = row['request_json'] + spec = store.submitted_spec('run-1') + assert not spec['accepted_plan'] + closed = {'workflow_id': 'delivery-run-1', 'execution_run_id': 'preplan-stop', + 'request_digest': row['request_digest'], 'recovery_digest': None, + 'result': {'run_id': 'run-1', 'phase': 'blocked', 'execution_state': 'blocked', + 'outcome': 'blocked', 'cleanup': 'confirmed', 'error': error, + 'revision': 3, 'iteration': 0, 'candidate': None, + 'pull_request': None, 'checks': {}}} + monkeypatch.setattr(DeliveryStore, '_completed_temporal_result', lambda *_a, **_k: closed) + command = {'continuation_kind': 'stopped_delivery_resume', 'command_id': 'preplan-resume', + 'expected_revision': 3, 'expected_iteration': 0, + 'expected_candidate_id': '0' * 64, 'expected_candidate_head': '0' * 40, + 'additional_iterations': 1} + + # Reopen the durable store to exercise the same boundary after a restart. + restarted = DeliveryStore(DeliveryConfig.load(initial.config.path)) + app = create_app(restarted.config.path) + app.state.delivery.store = restarted + with restarted._connect() as db: + before = list(db.iterdump()) + host = restarted.config.dashboard_url + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': 'stopped_delivery_resume', 'request': command}) + response = await client.post('/api/runs/run-1/continue-repair', json=command, + headers={'Origin': host, 'X-Devflow-CSRF': token}) + assert preview.status_code == 200 and not preview.json()['eligible'] + assert 'accepted_plan_unavailable' in {b['code'] for b in preview.json()['blockers']} + assert response.status_code == 409 and response.json()['detail'] == preview.json() + with restarted._connect() as db: + assert list(db.iterdump()) == before + assert db.execute('SELECT request_json FROM delivery_runs').fetchone()[0] == saved_request diff --git a/runtime/tests/test_delivery_unpublished_cleanup.py b/runtime/tests/test_delivery_unpublished_cleanup.py index c7113b1d..8bc27121 100644 --- a/runtime/tests/test_delivery_unpublished_cleanup.py +++ b/runtime/tests/test_delivery_unpublished_cleanup.py @@ -157,24 +157,62 @@ def command_for(state, candidate, *, unknown=False): 'additional_iterations': 1 if unknown else 0} -def test_public_unpublished_cleanup_then_existing_finite_continuation(closed_unpublished): +@pytest.mark.asyncio +async def test_public_unpublished_cleanup_then_existing_finite_continuation(closed_unpublished): + import httpx + + from devflow_temporal.delivery_api import create_app + store, broker, state, spec, calls, _external, shape = closed_unpublished original = copy.deepcopy(state) before = broker.candidate() + command = command_for(state, before, unknown=shape == 'unknown') + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + headers = {'Origin': host, 'X-Devflow-CSRF': token} + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + refused = await client.post('/api/runs/run-1/continue-repair', + json=command, headers=headers) + assert preview.status_code == 200 and not preview.json()['eligible'] + assert {'stopped_cleanup_unconfirmed', 'stopped_owner_conflict'} <= { + blocker['code'] for blocker in preview.json()['blockers']} + assert refused.status_code == 409 and refused.json()['detail'] == preview.json() + assert state == original and broker.candidate() == before + assert calls == [] assert reconcile_attempt(store, run_id='run-1', role='cleanup', iteration=0)['changed'] is True assert state == original and broker.candidate() == before assert len(calls) == 2 assert reconcile_attempt(store, run_id='run-1', role='cleanup', iteration=0)['changed'] is False assert len(calls) == 2 # No duplicate tracker effect and no successor admission. assert retry.retry_once(store) == [] # The sealed terminal classification remains terminal. - command = command_for(state, before, unknown=shape == 'unknown') preflight = store.repair_admission_preflight('run-1', command) assert preflight['authorized_through_iteration'] == command['additional_iterations'] with store._connect() as db: assert store.state.claim_for(db, spec['work_id']) is None assert db.execute('SELECT COUNT(*) FROM delivery_runs').fetchone()[0] == 1 assert not db.execute('SELECT 1 FROM delivery_repair_grants').fetchone() - admitted = store.continue_repair('run-1', command) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + headers = {'Origin': host, 'X-Devflow-CSRF': token} + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + assert preview.status_code == 200 and preview.json()['eligible'], preview.text + response = await client.post('/api/runs/run-1/continue-repair', + json=command, headers=headers) + assert response.status_code == 200, response.text + replay = await client.post('/api/runs/run-1/continue-repair', json=command, headers=headers) + assert replay.json() == response.json() + admitted = response.json() + # Idempotent replay returns the first admission without another owner or debit. + assert admitted['authorized_through_iteration'] == preflight['authorized_through_iteration'] + with store._connect() as db: + assert store.state.claim_for(db, spec['work_id'])['owner'] == 'external:devflow:run-1' + assert db.execute('SELECT COUNT(*) FROM delivery_runs').fetchone()[0] == 1 + assert not db.execute('SELECT 1 FROM delivery_repair_grants').fetchone() assert store.continue_repair('run-1', command) == admitted with store._connect() as db: recovery = json.loads(db.execute('SELECT recovery_json FROM delivery_runs').fetchone()[0]) diff --git a/runtime/tests/test_delivery_unused_budget_resume.py b/runtime/tests/test_delivery_unused_budget_resume.py index 0ba925b4..a5bf04ec 100644 --- a/runtime/tests/test_delivery_unused_budget_resume.py +++ b/runtime/tests/test_delivery_unused_budget_resume.py @@ -69,12 +69,30 @@ def test_unused_resume_keeps_the_original_failed_session_and_partial_source(fixe @pytest.mark.parametrize('iterations', [2, 3]) -def test_fixed_resume_cannot_extend_the_ceiling_or_prepare_runtime(fixed_stop, monkeypatch, - iterations): +@pytest.mark.asyncio +async def test_fixed_resume_cannot_extend_the_ceiling_or_prepare_runtime(fixed_stop, monkeypatch, + iterations): store, _, _, command = fixed_stop def unexpected(*_args, **_kwargs): pytest.fail('a budget extension reached runtime preparation') monkeypatch.setattr(resume, 'prepare_runtime', unexpected) + import httpx + + from devflow_temporal.delivery_api import create_app + + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + exhausted = {**command, 'additional_iterations': iterations} + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': exhausted}) + response = await client.post('/api/runs/run-1/continue-repair', json=exhausted, + headers={'Origin': host, 'X-Devflow-CSRF': token}) + assert preview.status_code == 200 and not preview.json()['eligible'] + assert preview.json()['blockers'][0]['code'] == 'product_budget_exhausted' + assert response.status_code == 409 and response.json()['detail'] == preview.json() with pytest.raises(ValueError, match='fixed repair budget'): store.continue_repair('run-1', {**command, 'additional_iterations': iterations}) with store._connect() as db: From 7132fb7233239587d23f3a251d142d8da0ff5cc2 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:10:37 +0100 Subject: [PATCH 03/24] fix(recovery): verify current checkpoint custody --- docs/durable-checkpoints.md | 15 +- .../devflow_temporal/delivery_checkpoints.py | 72 +++- .../test_delivery_checkpoint_native_escape.py | 148 ++++++++ runtime/tests/test_delivery_checkpoints.py | 338 +++++++++++++++++- 4 files changed, 567 insertions(+), 6 deletions(-) create mode 100644 runtime/tests/test_delivery_checkpoint_native_escape.py diff --git a/docs/durable-checkpoints.md b/docs/durable-checkpoints.md index 4169679e..3d46578f 100644 --- a/docs/durable-checkpoints.md +++ b/docs/durable-checkpoints.md @@ -93,6 +93,16 @@ The existing loopback API and CSRF boundary expose these operations: - `POST /api/runs/{run_id}/checkpoints/{checkpoint_id}/restore` accepts a new `command_id` and `expected_revision`. +Readback preserves the original `acknowledged_resumable` value as history. Its +current `resumable` observation also compares the admitted authority, run and +cumulative budget, issue owner, independent Git owner and candidate branch, +original resource manifest, and the current writer witness. If the checkout is +still present, it compares its retained source and index under that witness; +an absent checkout can still be restored from the sealed artifacts. Changed or +unavailable proof appears in `resume_blockers`. This is a read-only observation, +not a reservation or permission to resume: restoration must reacquire custody +and repeat its own admission checks. + Creation requires a stopped run, reconciled external effects, closed original launch and role leases, proven registered workspace ownership, and a fresh check for open workspace actors. Registered native journals currently cause a typed @@ -210,7 +220,10 @@ publication only when current fenced source/state reproduces its exact manifest and every artifact. Corrupt, incomplete, foreign or inconsistent bytes still refuse. Resource pressure or an interrupted write cannot produce a successful checkpoint receipt. Checkpoint directories live outside disposable -worktrees and have no automatic expiry or cleanup path. +worktrees and have no automatic expiry or cleanup path. Explicit retained-resource +pruning preserves checkpoint directories and durable session/evidence material; +a blocked candidate remains retained. No checkpoint expiration operation is +currently provided. Restoration validates every artifact and current execution authority, budget, owner, original resource generation and native closure again. It reconstructs a diff --git a/runtime/src/devflow_temporal/delivery_checkpoints.py b/runtime/src/devflow_temporal/delivery_checkpoints.py index 4712c5e3..324e4b75 100644 --- a/runtime/src/devflow_temporal/delivery_checkpoints.py +++ b/runtime/src/devflow_temporal/delivery_checkpoints.py @@ -1423,11 +1423,41 @@ def read_checkpoint(store, run_id, checkpoint_id): spec = store.effective_spec(run_id) with store._connect() as db: value, receipt = _read_acknowledged(db, spec, checkpoint_id) - current_effects = _frozen(db, _row(db, run_id))["delivery_effects"] + # Keep the tracking snapshot coherent without holding a SQL read + # transaction while the potentially large artifact inventory is hashed. + db.execute("BEGIN") + row = _row(db, run_id) + current = _frozen(db, row) + claim = store.state.claim_for(db, row["work_id"]) + owner = dict(claim) if claim else None + db.commit() # Readback retains the original ACK, but does not advertise a native # checkpoint as resumable under today's admission contract. Current custody # can have changed since the ACK; inability to inspect it is also unknown. blockers = list(value.get("resume_blockers", [])) + if digest(spec) != value["spec_sha256"]: + blockers.append({"scope": "authority", "reason": "checkpoint_authority_changed"}) + if not store.owns_execution(spec): + blockers.append({"scope": "authority", "reason": "checkpoint_transport_ownership_unknown"}) + if not _same_snapshot(current, value): + blockers.append({"scope": "run", "reason": "checkpoint_run_state_changed"}) + if owner != value["owner"]: + blockers.append({"scope": "owner", "reason": "checkpoint_execution_owner_changed"}) + try: + from .delivery_workspace_custody import source_for + + source = source_for(spec) + except (OSError, ValueError, subprocess.SubprocessError): + blockers.append({"scope": "source", "reason": "checkpoint_workspace_owner_unknown"}) + else: + try: + head = _git(source, "rev-parse", "refs/heads/" + value["source"]["branch"]) + except (OSError, ValueError, subprocess.SubprocessError): + blockers.append({"scope": "source", "reason": "checkpoint_candidate_branch_unknown"}) + else: + if head.decode().strip() != value["source"]["candidate"]["head"]: + blockers.append({"scope": "source", + "reason": "checkpoint_candidate_branch_changed"}) effect_blockers = [] feature_unknown = None current_feature = None @@ -1446,7 +1476,7 @@ def read_checkpoint(store, run_id, checkpoint_id): feature_unknown = "registry_observation_unknown" for scope, effects in ( ("delivery", value["frozen"]["delivery_effects"]), - ("delivery", current_effects), + ("delivery", current["delivery_effects"]), ("feature", (value.get("execution_ledger") or {}).get("execution_effects", [])), ("feature", (current_feature or {}).get("execution_effects", [])), ): @@ -1459,14 +1489,49 @@ def read_checkpoint(store, run_id, checkpoint_id): effect_blockers.append(blocker) if blocker not in blockers: blockers.append(blocker) + retained_source_unknown = False try: registry = RunResources(spec, read_only=True) with registry.locked(read_only=True) as resources: custody_unknown = bool(value.get("closed_journals") or resources["processes"]) + entry = resources["roots"].get(spec["checkout"]) + resource_changed = ( + not entry + or entry.get("kind") != "checkout" + or entry.get("identity") != value["checkout_identity"] + or _hash(read_private(registry.manifest, raw=True)) + != value["resource_manifest_sha256"] + ) + try: + # Follow creation/restoration's resource-before-writer lock order. + # The exclusive witness is observational and grants no later work. + with store._checkpoint_writer_context(spec): + writer_unknown = False + checkout = Path(spec["checkout"]) + if os.path.lexists(checkout) and _index_baseline_supported(value): + try: + from .delivery_retained_checkpoint import verify_retained_source + + _openers(checkout) + verify_retained_source( + spec, value, _root(spec) / checkpoint_id, resources) + except (OSError, ValueError, KeyError, sqlite3.Error, + subprocess.SubprocessError): + retained_source_unknown = True + except (OSError, ValueError, sqlite3.Error, subprocess.SubprocessError): + writer_unknown = True except (OSError, ValueError, KeyError): custody_unknown = True + resource_changed = True + writer_unknown = True if custody_unknown: blockers.append({"scope": "native", "reason": NativeCheckpointCustodyUnknown.code}) + if resource_changed: + blockers.append({"scope": "resource", "reason": "checkpoint_resource_custody_unknown"}) + if writer_unknown: + blockers.append({"scope": "writer", "reason": "checkpoint_writer_custody_unknown"}) + if retained_source_unknown: + blockers.append({"scope": "source", "reason": "checkpoint_retained_source_unverified"}) if feature_unknown: blockers.append({"scope": "feature", "reason": feature_unknown}) index_unknown = not _index_baseline_supported(value) @@ -1475,8 +1540,7 @@ def read_checkpoint(store, run_id, checkpoint_id): return { **receipt, "acknowledged_resumable": receipt["resumable"], - "resumable": (receipt["resumable"] and not custody_unknown - and not effect_blockers and not feature_unknown and not index_unknown), + "resumable": receipt["resumable"] and not blockers, "resume_blockers": blockers, "authority_sha256": value["spec_sha256"], "owner": value["owner"], diff --git a/runtime/tests/test_delivery_checkpoint_native_escape.py b/runtime/tests/test_delivery_checkpoint_native_escape.py new file mode 100644 index 00000000..de91ea46 --- /dev/null +++ b/runtime/tests/test_delivery_checkpoint_native_escape.py @@ -0,0 +1,148 @@ +"""An observed native monitor cannot prove every descendant has settled.""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest +from temporal_test_server import local_temporal +from temporalio import workflow +from temporalio.worker import UnsandboxedWorkflowRunner, Worker +from test_delivery_api import api_fixture as api_fixture +from test_native_interrupted_gate import native_gates + +from devflow_temporal.delivery_api import _dispatch_start, create_app +from devflow_temporal.delivery_native_process import NativeProcess, process_table +from devflow_temporal.delivery_resources import read_private +from devflow_temporal.delivery_writer_custody import to_thread + + +@workflow.defn(name="DevflowDeliveryWorkflow") +class NativeCheckpointOwner: + @workflow.run + async def run(self, spec, recovery=None): + return {"run_id": spec["run_id"], "execution_state": "blocked", + "outcome": "blocked", "iteration": 0} + + +@pytest.mark.asyncio +async def test_public_native_checkpoint_refuses_escaped_post_go_descendant( + api_fixture, tmp_path, +): + async with local_temporal() as env: + config_path, _request = api_fixture + raw = json.loads(config_path.read_text()) + raw["temporal_address"] = env.client.service_client.config.target_host + config_path.write_text(json.dumps(raw)) + fixture = native_gates.__wrapped__(api_fixture, tmp_path) + g = next(fixture) + ready, release = tmp_path / "escaped-ready.json", tmp_path / "escaped-release" + actor = None + try: + # The owned native body exits after a fast double fork. Its child + # closes inherited witnesses and stdio, reparents, and keeps no + # checkout cwd or open checkout file for lsof to observe. + code = """import json,os,sys,time +from pathlib import Path +from devflow_temporal.delivery_native_process import process_table +if os.fork(): os._exit(0) +os.setsid() +if os.fork(): os._exit(0) +os.closerange(0,1024) +os.chdir('/') +pid=os.getpid() +Path(sys.argv[1]).write_text(json.dumps({'pid':pid,'identity':process_table()[pid]['identity']})) +while not Path(sys.argv[2]).exists(): time.sleep(.02) +""" + folder = Path(g.spec["state_dir"]) / "owned-opaque-check" + process = NativeProcess( + g.spec, folder, argv=[sys.executable, "-I", "-c", code, str(ready), str(release)], + cwd=g.broker.checkout, timeout=20, + environment={"PATH": os.environ["PATH"], + "HOME": str(tmp_path / "owned-provider-home")}) + observed = await to_thread(g.spec, process.run) + deadline = time.monotonic() + 10 + while not ready.exists(): + assert time.monotonic() < deadline + await asyncio.sleep(.02) + actor = json.loads(ready.read_text()) + journal = read_private(folder / "native-process.json") + assert observed["cleanup"] == "observed-native-confirmed" + assert journal["monitoring_complete"] is True + assert str(actor["pid"]) not in journal["owned"] + assert process_table()[actor["pid"]]["identity"] == actor["identity"] + + # The exact original SDK execution closes; no mocked controller + # response or caller-provided custody assertion is used. + submitted = g.store.submitted_spec(g.spec["run_id"]) + async with Worker( + env.client, task_queue=g.store.config.queue, + workflow_runner=UnsandboxedWorkflowRunner(), workflows=[NativeCheckpointOwner], + ): + await _dispatch_start(SimpleNamespace(store=g.store, config=g.store.config), + env.client, g.store.pending_starts()[0], submitted) + assert (await env.client.get_workflow_handle( + "delivery-" + g.spec["run_id"]).result())["outcome"] == "blocked" + with g.store._connect() as db: + db.execute("UPDATE delivery_runs SET phase='blocked',execution_state='blocked'," + "outcome='blocked' WHERE run_id=?", (g.spec["run_id"],)) + revision = db.execute("SELECT revision FROM delivery_runs WHERE run_id=?", + (g.spec["run_id"],)).fetchone()[0] + url = g.store.config.dashboard_url.rstrip("/") + app = create_app(g.store.config.path) + path = "/api/runs/" + g.spec["run_id"] + "/checkpoints" + command = {"command_id": "escaped-native", "expected_revision": revision} + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url=url, + ) as client: + session = (await client.get("/api/session")).json() + headers = {"origin": url, "x-devflow-csrf": session["csrf_token"]} + response = await client.post(path, json=command, headers=headers) + assert response.status_code == 409, response.text + assert response.json()["detail"]["code"] == "native_checkpoint_custody_unknown" + assert process_table()[actor["pid"]]["identity"] == actor["identity"] + + # Even later disappearance cannot manufacture complete closure. + release.touch() + deadline = time.monotonic() + 10 + while True: + current = process_table().get(actor["pid"], {}) + if (current.get("identity") != actor["identity"] + or current.get("stat", "Z").startswith("Z")): + break + assert time.monotonic() < deadline + await asyncio.sleep(.02) + retry = await client.post(path, json=command, headers=headers) + assert retry.status_code == 409 + assert retry.json()["detail"]["code"] == "native_checkpoint_custody_unknown" + with g.store._connect() as db: + assert db.execute("SELECT 1 FROM delivery_commands WHERE command_id=?", + ("checkpoint-create:escaped-native",)).fetchone() is None + assert not list((Path(g.spec["state_dir"]) / "checkpoints").glob("*/manifest.json")) + finally: + release.touch() + if actor: + current = process_table().get(actor["pid"], {}) + if (current.get("identity") == actor["identity"] + and not current.get("stat", "Z").startswith("Z")): + try: + os.kill(actor["pid"], signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + 5 + while True: + current = process_table().get(actor["pid"], {}) + if (current.get("identity") != actor["identity"] + or current.get("stat", "Z").startswith("Z")): + break + assert time.monotonic() < deadline + await asyncio.sleep(.02) + next(fixture, None) diff --git a/runtime/tests/test_delivery_checkpoints.py b/runtime/tests/test_delivery_checkpoints.py index 65a4b691..13c7d8e9 100644 --- a/runtime/tests/test_delivery_checkpoints.py +++ b/runtime/tests/test_delivery_checkpoints.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import multiprocessing import os @@ -9,7 +10,7 @@ import sqlite3 import subprocess import sys -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from pathlib import Path import httpx @@ -521,7 +522,55 @@ def test_selected_session_bytes_survive_source_deletion(stopped): @pytest.mark.asyncio async def test_public_checkpoint_routes_require_csrf_and_return_verified_readback(stopped): + from devflow_temporal.contracts import digest + from devflow_temporal.delivery_continuation import session_state_digest + store, spec, broker, row = stopped + # A passed fake implementation has an unpublished commit, then separate + # staged, unstaged and untracked work when a check stops the delivery. + git(broker.checkout, "commit", "-qm", "Owned unpublished implementation") + (broker.checkout / "README.md").write_text("staged after implementation\n") + git(broker.checkout, "add", "README.md") + (broker.checkout / "README.md").write_text("staged plus unfinished work\n") + candidate = broker.candidate() + staged = git(broker.checkout, "diff", "--cached") + assert candidate["head"] != spec["base_sha"] + assert not git(Path(spec["origin_url"]), "for-each-ref", "refs/heads/" + spec["branch"]) + + state = Path(spec["state_dir"]) + session_id = "owned-implementation-session" + home = state / "role-homes/implement" + sessions = home / "codex/sessions" + sessions.mkdir(parents=True, mode=0o700) + transcript = sessions / (session_id + ".jsonl") + transcript.write_text('{"fixture":"implementation transcript"}\n') + transcript.chmod(0o600) + (home / "codex/auth.json").write_text("private fixture credential") + session_digest = session_state_digest(home, session_id) + evidence = state / "evidence" / "stopped-check.xml" + evidence.parent.mkdir(mode=0o700) + evidence.write_text("\n") + evidence.chmod(0o600) + evidence_sha256 = hashlib.sha256(evidence.read_bytes()).hexdigest() + job_key = digest({"owned": "passed-implementation"}) + request = {"spec": spec, "role": "implement", "iteration": 1, "candidate": candidate} + write_private(state / "attempts" / job_key / "request.json", request) + result = {"role": "implement", "iteration": 1, "status": "pass", "candidate": candidate, + "session_id": session_id, "cleanup": "confirmed"} + checks = {"prepublish": {"state": "failed", "candidate_id": candidate["id"], + "results": [{"id": "owned-check", "passed": False, + "cleanup": "confirmed", "path": str(evidence), + "sha256": evidence_sha256}]}} + with store._connect() as db: + db.execute("INSERT INTO delivery_attempts " + "(job_key,run_id,role,iteration,candidate_id,state,session_id,result_json," + "cleanup) VALUES (?,?,'implement',1,?,'finished',?,?,'confirmed')", + (job_key, spec["run_id"], candidate["id"], session_id, json.dumps(result))) + db.execute("UPDATE delivery_runs SET candidate_json=?,checks_json=? WHERE run_id=?", + (json.dumps(candidate), json.dumps(checks), spec["run_id"])) + row = dict(db.execute("SELECT * FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone()) + claim = dict(store.state.claim_for(db, spec["work_id"])) app = create_app(store.config.path) url = store.config.dashboard_url.rstrip("/") async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=url) as client: @@ -534,8 +583,12 @@ async def test_public_checkpoint_routes_require_csrf_and_return_verified_readbac assert response.status_code == 200, response.text receipt = response.json() assert (await client.get(path + "/" + receipt["checkpoint_id"])).json()["resumable"] + checkpoint = state / "checkpoints" / receipt["checkpoint_id"] + assert not list(checkpoint.rglob("auth.json")) shutil.rmtree(Path(spec["source_path"])) shutil.rmtree(broker.checkout) + shutil.rmtree(home) + evidence.unlink() response = await client.post( path + "/" + receipt["checkpoint_id"] + "/restore", json={"command_id": "http-restore", "expected_revision": row["revision"]}, @@ -543,6 +596,289 @@ async def test_public_checkpoint_routes_require_csrf_and_return_verified_readbac ) assert response.status_code == 200, response.text assert response.json()["execution_started"] is False + assert response.json()["candidate"] == candidate_for(broker.checkout) + assert broker.candidate() == candidate + assert git(broker.checkout, "diff", "--cached") == staged + assert (broker.checkout / "README.md").read_text() == "staged plus unfinished work\n" + assert (broker.checkout / "owned-untracked.txt").read_text() == "recover me\n" + assert session_state_digest(home, session_id) == session_digest + assert hashlib.sha256(evidence.read_bytes()).hexdigest() == evidence_sha256 + assert not (home / "codex/auth.json").exists() + with store._connect() as db: + assert dict(db.execute("SELECT * FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone()) == row + assert dict(store.state.claim_for(db, spec["work_id"])) == claim + + +@pytest.mark.asyncio +async def test_public_recovery_continues_restored_stopped_gate(stopped, monkeypatch): + """Recovery must use the same stopped candidate after both fixture checkouts vanish.""" + from devflow_temporal.contracts import digest + from devflow_temporal.delivery_resource_maintenance import prune + + store, spec, broker, _row = stopped + # Keep this continuation inside its admitted source scope. The fake role and + # closed workflow are fixtures; source, cleanup, checkpoint and public + # recovery admission use their actual implementations. + (broker.checkout / "owned-untracked.txt").unlink() + (broker.checkout / ".gitignore").unlink() + (broker.checkout / "auth.json").unlink() + git(broker.checkout, "commit", "-qm", "Fixture unpublished implementation") + (broker.checkout / "README.md").write_text("Unfinished implementation\n") + candidate = broker.candidate() + state_dir = Path(spec["state_dir"]) + session_id = "owned-implementation-session" + home = state_dir / "role-homes/implement" + sessions = home / "codex/sessions" + sessions.mkdir(mode=0o700, parents=True) + transcript = sessions / (session_id + ".jsonl") + transcript.write_text('{"owned":"fixture"}\n') + transcript.chmod(0o600) + job_key = digest({"owned": "implementation"}) + request = {"spec": spec, "role": "implement", "iteration": 1, "candidate": candidate} + write_private(state_dir / "attempts" / job_key / "request.json", request) + implementation = {"role": "implement", "iteration": 1, "status": "pass", + "session_id": session_id, "cleanup": "confirmed", "candidate": candidate} + cleanup = RunResources(spec).finalize("blocked") + assert cleanup["state"] == "confirmed", cleanup + checks = {"prepublish": {"state": "failed", "source_unchanged": True, + "candidate_id": candidate["id"], "results": [ + {"id": "packaging", "passed": False, + "cleanup": "confirmed", "exit_code": 1}]}, + "resource_cleanup": cleanup} + state = {"run_id": spec["run_id"], "revision": 9, "iteration": 1, "phase": "blocked", + "execution_state": "blocked", "outcome": "blocked", "cleanup": "confirmed", + "error": "prepublication repair limit exhausted", "candidate": candidate, + "pull_request": None, "candidate_revision": 2, "roles": [implementation], + "checks": checks, "usage": {"tokens": 419, "wall_seconds": 12}, + "findings": ["Historical packaging failure"]} + store.project(spec["run_id"], phase="blocked", execution_state="blocked", + event_type="blocked", message=state["error"], candidate=candidate, + checks=checks, usage=state["usage"], iteration=1, protocol_revision=9, + outcome="blocked", cleanup="confirmed", error=state["error"]) + with store._connect() as db: + db.execute("INSERT INTO delivery_attempts " + "(job_key,run_id,role,iteration,candidate_id,state,session_id,result_json," + "cleanup) VALUES (?,?,'implement',1,?,'finished',?,?,'confirmed')", + (job_key, spec["run_id"], candidate["id"], session_id, + json.dumps(implementation))) + db.execute("UPDATE delivery_runs SET workflow_id=? WHERE run_id=?", + ("delivery-" + spec["run_id"], spec["run_id"])) + store.state.release_work(db, spec["work_id"], "external:devflow:" + spec["run_id"]) + row = dict(db.execute("SELECT * FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone()) + closed = {"workflow_id": "delivery-" + spec["run_id"], "execution_run_id": "closed-original", + "request_digest": spec["request_digest"], "recovery_digest": None, + "result": state, "closed_at": "2026-01-01T00:00:00+00:00"} + monkeypatch.setattr(store, "_completed_temporal_result", lambda *_args, **_kw: closed) + monkeypatch.setattr(store, "_ensure_no_remote_pr", lambda *_: None) + app = create_app(store.config.path) + app.state.delivery.store = store + url = store.config.dashboard_url.rstrip("/") + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=url) as client: + headers = {"origin": url, "x-devflow-csrf": (await client.get("/api/session")).json()[ + "csrf_token"]} + path = "/api/runs/" + spec["run_id"] + "/checkpoints" + saved = await client.post(path, headers=headers, json={ + "command_id": "integration-save", "expected_revision": row["revision"]}) + assert saved.status_code == 200, saved.text + pruned = prune(store, spec["run_id"]) + assert any(item["path"] == str(broker.checkout) and item["state"] == "retained" + for item in pruned["roots"]) + retained = await client.get(path + "/" + saved.json()["checkpoint_id"]) + assert retained.status_code == 200 and retained.json()["resumable"] + assert retained.json()["manifest_sha256"] == saved.json()["manifest_sha256"] + shutil.rmtree(Path(spec["source_path"])) + shutil.rmtree(broker.checkout) + shutil.rmtree(home) + restored = await client.post(path + "/" + saved.json()["checkpoint_id"] + "/restore", + headers=headers, json={"command_id": "integration-restore", + "expected_revision": row["revision"]}) + assert restored.status_code == 200, restored.text + assert broker.candidate() == candidate and transcript.read_text() == '{"owned":"fixture"}\n' + command = {"continuation_kind": "stopped_delivery_resume", "command_id": "resume-1", + "expected_revision": 9, "expected_iteration": 1, + "expected_candidate_id": candidate["id"], + "expected_candidate_head": candidate["head"], "additional_iterations": 1} + admitted = await client.post("/api/runs/" + spec["run_id"] + "/continue-repair", + headers=headers, json=command) + assert admitted.status_code == 200, admitted.text + assert admitted.json()["run_id"] == spec["run_id"] + assert admitted.json()["authorized_through_iteration"] == 2 + replay = await client.post("/api/runs/" + spec["run_id"] + "/continue-repair", + headers=headers, json=command) + assert replay.status_code == 200 and replay.json() == admitted.json() + with store._connect() as db: + current = db.execute("SELECT * FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone() + recovery = json.loads(current["recovery_json"]) + assert recovery["session_id"] == session_id + assert recovery["state"]["checks"] == checks + assert recovery["state"]["usage"] == state["usage"] + assert recovery["candidate"] == candidate + assert db.execute("SELECT count(*) FROM delivery_runs").fetchone()[0] == 1 + assert store.state.claim_for(db, spec["work_id"])["owner"] == ( + "external:devflow:" + spec["run_id"]) + assert not db.execute("SELECT 1 FROM delivery_effects WHERE kind='publish'").fetchone() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drift", ["budget", "authority", "owner", "workspace_owner", "resource", + "writer", "branch", "source", "index"]) +async def test_public_readback_refuses_stale_custody_without_rewriting_ack(stopped, drift): + """A historical ACK must not advertise recovery after its current proof diverges.""" + from devflow_temporal.delivery_writer_custody import shared + + store, spec, broker, row = stopped + if drift == "branch": + git(broker.checkout, "commit", "-qm", "Owned unpublished fixture") + app = create_app(store.config.path) + app.state.delivery.store = store + url = store.config.dashboard_url.rstrip("/") + path = "/api/runs/" + spec["run_id"] + "/checkpoints" + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=url) as client: + token = (await client.get("/api/session")).json()["csrf_token"] + headers = {"origin": url, "x-devflow-csrf": token} + saved = await client.post(path, headers=headers, json={ + "command_id": "current-custody-save", "expected_revision": row["revision"]}) + assert saved.status_code == 200, saved.text + receipt = saved.json() + checkpoint_path = path + "/" + receipt["checkpoint_id"] + assert (await client.get(checkpoint_path)).json()["resumable"] + if drift == "budget": + with store._connect() as db: + db.execute("UPDATE delivery_runs SET usage_json=? WHERE run_id=?", + ('{"tokens":420}', spec["run_id"])) + reason = "checkpoint_run_state_changed" + elif drift == "authority": + with store._connect() as db: + db.execute("UPDATE delivery_runs SET accepted_plan_text=? WHERE run_id=?", + ("A different accepted plan", spec["run_id"])) + reason = "checkpoint_authority_changed" + elif drift == "owner": + with store._connect() as db: + store.state.release_work(db, spec["work_id"], "external:devflow:" + spec["run_id"]) + reason = "checkpoint_execution_owner_changed" + elif drift == "workspace_owner": + owner = Path(spec["state_dir"]) / "workspace-owner.git" + owner.rename(owner.with_name("fixture-owner-unavailable.git")) + reason = "checkpoint_workspace_owner_unknown" + elif drift == "resource": + registry = RunResources(spec, read_only=True) + resources = read_private(registry.manifest) + resources["roots"][str(broker.checkout)]["generation"] = 42 + write_private(registry.manifest, resources) + reason = "checkpoint_resource_custody_unknown" + elif drift == "branch": + git(broker.checkout, "update-ref", "refs/heads/" + spec["branch"], spec["base_sha"]) + reason = "checkpoint_candidate_branch_changed" + elif drift == "source": + (broker.checkout / "README.md").write_text("new unsaved stopped progress\n") + reason = "checkpoint_retained_source_unverified" + elif drift == "index": + git(broker.checkout, "update-index", "--assume-unchanged", "README.md") + reason = "checkpoint_retained_source_unverified" + else: + reason = "checkpoint_writer_custody_unknown" + + with shared(spec) if drift == "writer" else nullcontext(): + if drift == "writer": + (broker.checkout / "README.md").write_text("new unsaved writer progress\n") + observed = await client.get(checkpoint_path) + assert observed.status_code == 200, observed.text + readback = observed.json() + assert readback["acknowledged_resumable"] is True + assert readback["resumable"] is False + assert reason in {item["reason"] for item in readback["resume_blockers"]} + assert readback["manifest_sha256"] == receipt["manifest_sha256"] + if drift == "writer": + refused = await client.post(path, headers=headers, json={ + "command_id": "live-writer", "expected_revision": row["revision"]}) + assert refused.status_code == 409 + if drift == "writer": + assert (broker.checkout / "README.md").read_text() == "new unsaved writer progress\n" + return + if drift in {"source", "index"}: + assert broker.checkout.exists() + if drift == "source": + assert (broker.checkout / "README.md").read_text() == ( + "new unsaved stopped progress\n") + return + shutil.rmtree(broker.checkout) + restored = await client.post(checkpoint_path + "/restore", headers=headers, json={ + "command_id": "stale-custody-restore", "expected_revision": row["revision"]}) + assert restored.status_code == 409 + assert not broker.checkout.exists() + with store._connect() as db: + assert db.execute( + "SELECT count(*) FROM delivery_commands WHERE command_id=?", + ("checkpoint-restore:stale-custody-restore",), + ).fetchone()[0] == 0 + + +@pytest.mark.asyncio +async def test_public_historical_checkpoint_refuses_new_native_custody(stopped): + """An older ACK remains evidence when current native closure is unknown.""" + store, spec, broker, row = stopped + receipt = create_checkpoint(store, spec["run_id"], "before-native", row["revision"]) + registry = RunResources(spec, read_only=True) + resources = read_private(registry.manifest) + # Model a newly registered native journal after an existing acknowledgement. + resources["processes"].append(str(Path(spec["state_dir"]) / "native-process.json")) + write_private(registry.manifest, resources) + app = create_app(store.config.path) + url = store.config.dashboard_url.rstrip("/") + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=url) as client: + path = "/api/runs/" + spec["run_id"] + "/checkpoints" + token = (await client.get("/api/session")).json()["csrf_token"] + headers = {"origin": url, "x-devflow-csrf": token} + readback = (await client.get(path + "/" + receipt["checkpoint_id"])).json() + assert readback["acknowledged_resumable"] is True + assert readback["resumable"] is False + assert "native_checkpoint_custody_unknown" in { + item["reason"] for item in readback["resume_blockers"]} + shutil.rmtree(broker.checkout) + response = await client.post(path + "/" + receipt["checkpoint_id"] + "/restore", + headers=headers, json={"command_id": "native-restore", + "expected_revision": row["revision"]}) + assert response.status_code == 409 + assert response.json()["detail"]["code"] == "native_checkpoint_custody_unknown" + assert not broker.checkout.exists() + assert not list(broker.checkout.parent.glob(".checkpoint-restore-*")) + with store._connect() as db: + assert db.execute("SELECT 1 FROM delivery_mutations WHERE command_id=?", + ("checkpoint-restore:native-restore",)).fetchone() is None + + +def test_fenced_checkpoint_preserves_unknown_effect_without_restoration(stopped): + """A trusted fence can retain an uncertain effect but cannot replay it.""" + store, spec, broker, row = stopped + with store._connect() as db: + db.execute("INSERT INTO delivery_effects VALUES (?,?,?,?,'pending',NULL,'now')", + ("unknown-publish", spec["run_id"], "publish", "{}")) + fence = durable_fence(store, spec) + with pytest.raises(ValueError, match="authenticated pause fence"): + create_checkpoint(store, spec["run_id"], "unknown-without-fence", row["revision"], + allow_unresolved_effects=True) + receipt = create_checkpoint(store, spec["run_id"], "unknown-with-fence", row["revision"], + fence=fence, allow_unresolved_effects=True) + assert not receipt["resumable"] + assert receipt["resume_blockers"] == [{"scope": "delivery", + "reason": "external_outcome_unknown", "effect_key": "unknown-publish", + "kind": "publish", "state": "pending"}] + readback = read_checkpoint(store, spec["run_id"], receipt["checkpoint_id"]) + assert not readback["resumable"] and readback["pending_effects"] + shutil.rmtree(broker.checkout) + with pytest.raises(ValueError, match="require reconciliation"): + restore_checkpoint(store, spec["run_id"], receipt["checkpoint_id"], "unknown-restore", + row["revision"], fence=fence) + assert not broker.checkout.exists() + with store._connect() as db: + effect = db.execute("SELECT state,observed_json FROM delivery_effects " + "WHERE effect_key='unknown-publish'").fetchone() + assert tuple(effect) == ("pending", None) + assert db.execute("SELECT 1 FROM delivery_mutations WHERE command_id=?", + ("checkpoint-restore:unknown-restore",)).fetchone() is None @pytest.mark.asyncio From 3e1314ea57156bd5ec8020a43680a7226bd84807 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:13:30 +0100 Subject: [PATCH 04/24] fix(recovery): preserve assessment owner context --- .../devflow_temporal/delivery_lifecycle.py | 10 ++- .../delivery_stopped_resume.py | 85 +++++++++++-------- .../test_delivery_feature_resumed_evidence.py | 10 +++ runtime/tests/test_delivery_gate_retry.py | 18 ++++ .../test_delivery_store_lifecycle_240.py | 31 +++++++ 5 files changed, 113 insertions(+), 41 deletions(-) diff --git a/runtime/src/devflow_temporal/delivery_lifecycle.py b/runtime/src/devflow_temporal/delivery_lifecycle.py index bab60eb9..5eab1e65 100644 --- a/runtime/src/devflow_temporal/delivery_lifecycle.py +++ b/runtime/src/devflow_temporal/delivery_lifecycle.py @@ -23,7 +23,7 @@ # checkpoint and rechecks it when acquiring admission custody. The table does # not infer a route from an error or authorize a transition by itself. TRANSITION_TABLE = { - kind: {'from': ['stopped'], 'to': 'recovering'} for kind in ( + kind: {'from': ('stopped',), 'to': 'recovering'} for kind in ( 'repair_continuation', 'stopped_delivery_resume', 'published_gate_retry', 'prepublication_gate_retry', 'published_check_prelaunch_retry', 'published_ci_retry', 'published_controller_retry', 'recover_publication', @@ -33,7 +33,7 @@ } # Cancelled feature workers and revision-owned completed workers can be resumed # only when their owner supplies the additional historical authority. -TRANSITION_TABLE['stopped_delivery_resume']['from'].append('terminal') +TRANSITION_TABLE['stopped_delivery_resume']['from'] = ('stopped', 'terminal') def lifecycle_from_row(row: Mapping[str, Any]) -> str: @@ -43,7 +43,7 @@ def lifecycle_from_row(row: Mapping[str, Any]) -> str: return 'stopped' if execution == 'terminal' and outcome in {'delivered', 'cancelled', 'failed'}: return 'terminal' - if execution == 'stopping': + if execution in {'stopping', 'cancelling'}: return 'stopping' if execution in {'queued', 'pending', 'running'}: if row.get('recovery_json'): @@ -138,11 +138,13 @@ def block(code, fact, action, authorization=False, satisfied=None): 'satisfied': predicate.get('satisfied'), 'user_input_required': predicate.get('user_input_required'), }) + transition = TRANSITION_TABLE.get(operation) observed = {key: facts.get(key) for key in ( 'revision', 'iteration', 'candidate_id', 'candidate_head', 'publication_number')} return {'version': VERSION, 'operation': operation, 'normalized_lifecycle': facts.get('normalized_lifecycle', 'unknown'), - 'transition': TRANSITION_TABLE.get(operation), + 'transition': ({'from': list(transition['from']), 'to': transition['to']} + if transition is not None else None), 'eligible': not blockers, 'observed_revision': observed['revision'], 'candidate': {'id': observed['candidate_id'], 'head': observed['candidate_head']}, 'publication': ({'number': observed['publication_number']} diff --git a/runtime/src/devflow_temporal/delivery_stopped_resume.py b/runtime/src/devflow_temporal/delivery_stopped_resume.py index 6557c980..34ab8aeb 100644 --- a/runtime/src/devflow_temporal/delivery_stopped_resume.py +++ b/runtime/src/devflow_temporal/delivery_stopped_resume.py @@ -1117,7 +1117,11 @@ def _snapshot(store, run_id, *, _revision=None, review_subject=False, _observati state_checks, row_checks = closed_checks(row, state, cleanup_authority) # Keep the closed verdict immutable while reporting later custody facts # separately. These are exactly the facts checked by the original guard. - from .delivery_lifecycle import lifecycle_from_row, require_observed_predicates + from .delivery_lifecycle import ( + RecoveryRefused, + lifecycle_from_row, + require_observed_predicates, + ) status_matches = (row['phase'] in stopped and row['outcome'] == row['phase'] and row['execution_state'] == expected_execution and state.get('run_id') == run_id @@ -1137,42 +1141,49 @@ def _snapshot(store, run_id, *, _revision=None, review_subject=False, _observati and bool(spec['accepted_plan'].strip())) effects_settled = all(e['state'] == 'complete' and e['observed_json'] is not None for e in effects) - require_observed_predicates(KIND, 'delivery_stopped_resume', {'row': row, 'closed': closed}, [ - {'code': 'stopped_state_unproven', 'satisfied': status_matches, - 'missing_or_unknown_fact': 'matching stopped or supported terminal run state', - 'next_action': 'inspect the retained run and closed predecessor', - 'user_input_required': None}, - {'code': 'stopped_cleanup_unconfirmed', 'satisfied': True if cleanup_matches else None, - 'missing_or_unknown_fact': 'confirmed cleanup or authenticated later reconciliation', - 'next_action': 'reconcile the original cleanup and custody', - 'user_input_required': None}, - {'code': 'stopped_predecessor_unproven', 'satisfied': identity_matches, - 'missing_or_unknown_fact': 'matching workflow, admission and recovery identities', - 'next_action': 'inspect the closed predecessor and its original admission', - 'user_input_required': None}, - {'code': 'stopped_checkpoint_changed', 'satisfied': checkpoint_matches, - 'missing_or_unknown_fact': 'unchanged candidate, checks, publication and verdict', - 'next_action': 'inspect retained checkpoint and immutable history', - 'user_input_required': None}, - {'code': 'stopped_owner_conflict', 'satisfied': claim is None, - 'missing_or_unknown_fact': 'released original work claim', - 'next_action': 'inspect and reconcile the current work owner', - 'user_input_required': None}, - {'code': 'stopped_effects_unsettled', 'satisfied': True if effects_settled else None, - 'missing_or_unknown_fact': 'settled effects with observed outcomes', - 'next_action': 'reconcile the original effects', 'user_input_required': None}, - {'code': 'accepted_plan_unavailable', 'satisfied': plan_available, - 'missing_or_unknown_fact': 'original accepted plan for implementation resume', - 'next_action': 'inspect retained intake and preparation; no implementation resume ' - 'is authorized without an accepted plan', - 'user_input_required': None}, - ], lifecycle=lifecycle_from_row(row), owner_proof=( - True if (status_matches and cleanup_matches and identity_matches and checkpoint_matches - and claim is None and effects_settled) else None), - legacy_message=('resume requires a closed finalized delivery with released ownership' - if not (status_matches and cleanup_matches and identity_matches - and checkpoint_matches and claim is None and effects_settled) - else 'implementation resume requires an accepted plan')) + try: + require_observed_predicates( + KIND, 'delivery_stopped_resume', {'row': row, 'closed': closed}, [ + {'code': 'stopped_state_unproven', 'satisfied': status_matches, + 'missing_or_unknown_fact': 'matching stopped or supported terminal run state', + 'next_action': 'inspect the retained run and closed predecessor', + 'user_input_required': None}, + {'code': 'stopped_cleanup_unconfirmed', 'satisfied': True if cleanup_matches else None, + 'missing_or_unknown_fact': 'confirmed cleanup or authenticated later reconciliation', + 'next_action': 'reconcile the original cleanup and custody', + 'user_input_required': None}, + {'code': 'stopped_predecessor_unproven', 'satisfied': identity_matches, + 'missing_or_unknown_fact': 'matching workflow, admission and recovery identities', + 'next_action': 'inspect the closed predecessor and its original admission', + 'user_input_required': None}, + {'code': 'stopped_checkpoint_changed', 'satisfied': checkpoint_matches, + 'missing_or_unknown_fact': 'unchanged candidate, checks, publication and verdict', + 'next_action': 'inspect retained checkpoint and immutable history', + 'user_input_required': None}, + {'code': 'stopped_owner_conflict', 'satisfied': claim is None, + 'missing_or_unknown_fact': 'released original work claim', + 'next_action': 'inspect and reconcile the current work owner', + 'user_input_required': None}, + {'code': 'stopped_effects_unsettled', 'satisfied': True if effects_settled else None, + 'missing_or_unknown_fact': 'settled effects with observed outcomes', + 'next_action': 'reconcile the original effects', 'user_input_required': None}, + {'code': 'accepted_plan_unavailable', 'satisfied': plan_available, + 'missing_or_unknown_fact': 'original accepted plan for implementation resume', + 'next_action': 'inspect retained intake and preparation; no implementation resume ' + 'is authorized without an accepted plan', + 'user_input_required': None}, + ], lifecycle=lifecycle_from_row(row), owner_proof=( + True if (status_matches and cleanup_matches and identity_matches and checkpoint_matches + and claim is None and effects_settled) else None), + legacy_message=('resume requires a closed finalized delivery with released ownership' + if not (status_matches and cleanup_matches and identity_matches + and checkpoint_matches and claim is None and effects_settled) + else 'implementation resume requires an accepted plan')) + except RecoveryRefused as exc: + if review_subject: + # Other owners must report their own operation, not a resume refusal. + raise ValueError(str(exc)) from exc + raise if spec.get('baseline_checks_version') in (1, 2): baseline = state.get('checks', {}).get('baseline', {}) required = {check['id'] for check in spec['policy']['baseline_checks']} diff --git a/runtime/tests/test_delivery_feature_resumed_evidence.py b/runtime/tests/test_delivery_feature_resumed_evidence.py index 973d6064..373b6e9b 100644 --- a/runtime/tests/test_delivery_feature_resumed_evidence.py +++ b/runtime/tests/test_delivery_feature_resumed_evidence.py @@ -221,6 +221,16 @@ def test_stale_older_resume_link_blocks_only_implementations_that_ran_under_it( role = codex_receipt(store, child, source=source, output=output) cancelled(store, monkeypatch, parent, child, role, output) row = {"error": None, "checks_json": "{}"} + original_admit = resume_mod.admit + + def observed_admission(owner, run_id, command, **kwargs): + preview = original_admit(owner, run_id, command, preflight=True, **kwargs) + assessment = preview["lifecycle_assessment"] + assert assessment["eligible"] and assessment["normalized_lifecycle"] == "terminal" + assert assessment["transition"]["from"] == ["stopped", "terminal"] + return original_admit(owner, run_id, command, **kwargs) + + monkeypatch.setattr(resume_mod, "admit", observed_admission) first = activities.resume_worker(store, parent, child, row) with store._connect() as db: older = json.loads(db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id=?", diff --git a/runtime/tests/test_delivery_gate_retry.py b/runtime/tests/test_delivery_gate_retry.py index 3efc2d41..83ed4c0e 100644 --- a/runtime/tests/test_delivery_gate_retry.py +++ b/runtime/tests/test_delivery_gate_retry.py @@ -293,7 +293,25 @@ async def assess(): headers=headers) assert not owned.json()['eligible'] assert 'gate_owner_conflict' in {b['code'] for b in owned.json()['blockers']} + assert owned.json()['proof_digest'] is None assert response.status_code == 409 and response.json()['detail'] == owned.json() + # A diagnostic that observes this stopped subject owns a different route. + from devflow_temporal import delivery_native_diagnostic as diagnostic + + monkeypatch.setattr(diagnostic, '_readonly', lambda value: value) + diagnostic_request = { + 'command_id': 'diagnostic', 'expected_revision': state['revision'], + 'expected_projection_revision': 0, 'expected_iteration': state['iteration'], + 'expected_candidate_id': command['expected_candidate_id'], + 'expected_candidate_head': command['expected_candidate_head'], + 'expected_content_sha256': '0' * 64, 'expected_pr_number': 1, + 'expected_pr_head': '0' * 40, 'check_id': diagnostic.CHECK_ID, + 'expected_check_sha256': diagnostic.CHECK_SHA256, + } + diagnostic_result = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': diagnostic.KIND, 'request': diagnostic_request}) + assert not diagnostic_result.json()['eligible'] + assert diagnostic_result.json()['operation'] == diagnostic.KIND with store._connect() as db: assert not db.execute('SELECT 1 FROM delivery_gate_admissions').fetchone() store.state.release_work(db, broker.spec['work_id'], 'external:other') diff --git a/runtime/tests/test_delivery_store_lifecycle_240.py b/runtime/tests/test_delivery_store_lifecycle_240.py index 73148a11..d3add79d 100644 --- a/runtime/tests/test_delivery_store_lifecycle_240.py +++ b/runtime/tests/test_delivery_store_lifecycle_240.py @@ -63,17 +63,25 @@ async def test_default_repair_public_assessment_and_admission_agree(service, mon @pytest.mark.asyncio async def test_preplan_stop_refuses_implementation_resume_across_restart(service, monkeypatch): """A closed pre-intake preparation failure must not be resumed as implementation.""" + import asyncio import copy import json + import socket + import sys import httpx + import uvicorn from devflow_temporal.delivery_api import create_app from devflow_temporal.delivery_config import DeliveryConfig from devflow_temporal.delivery_store import DeliveryStore initial, request = service + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + listener.listen(128) raw = copy.deepcopy(initial.config.raw) + raw['dashboard_url'] = f'http://127.0.0.1:{listener.getsockname()[1]}' raw['roles']['intake'] = {'model': 'fixture', 'effort': 'low'} initial.config.path.write_text(json.dumps(raw)) store = DeliveryStore(DeliveryConfig.load(initial.config.path)) @@ -119,6 +127,29 @@ async def test_preplan_stop_refuses_implementation_resume_across_restart(service assert preview.status_code == 200 and not preview.json()['eligible'] assert 'accepted_plan_unavailable' in {b['code'] for b in preview.json()['blockers']} assert response.status_code == 409 and response.json()['detail'] == preview.json() + expected_assessment = preview.json() + # Exercise the real read-only CLI and its loopback client against this owned API. + assessment_file = initial.config.path.parent / 'preplan-assessment.json' + assessment_file.write_text(json.dumps({'kind': 'stopped_delivery_resume', 'request': command})) + server = uvicorn.Server(uvicorn.Config(app, lifespan='off', log_level='error')) + serving = asyncio.create_task(server.serve(sockets=[listener])) + try: + for _ in range(200): + if server.started: + break + await asyncio.sleep(.01) + assert server.started + process = await asyncio.create_subprocess_exec( + sys.executable, '-m', 'devflow_temporal.delivery_control', '--config', + str(initial.config.path), 'assess-transition', '--id', 'run-1', '--request', + str(assessment_file), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + stdout, stderr = await asyncio.wait_for(process.communicate(), 30) + assert process.returncode == 0, stderr.decode() + assert json.loads(stdout) == expected_assessment + finally: + server.should_exit = True + await asyncio.wait_for(serving, 10) + listener.close() with restarted._connect() as db: assert list(db.iterdump()) == before assert db.execute('SELECT request_json FROM delivery_runs').fetchone()[0] == saved_request From 98386ebbfd1e5fd5703d9250e47f8b2cdae2f790 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:18:30 +0100 Subject: [PATCH 05/24] fix(recovery): retain candidates after readiness failures --- docs/trusted-local-delivery.md | 18 + .../devflow_temporal/delivery_activities.py | 12 +- .../delivery_automatic_retry.py | 13 + .../devflow_temporal/delivery_gate_retry.py | 122 ++- .../delivery_readiness_gate_retry.py | 127 +++ .../src/devflow_temporal/delivery_workflow.py | 10 + .../tests/test_delivery_automatic_retry.py | 15 +- ...test_delivery_feature_readiness_handoff.py | 195 +++++ .../test_delivery_readiness_gate_retry.py | 739 ++++++++++++++++++ 9 files changed, 1225 insertions(+), 26 deletions(-) create mode 100644 runtime/src/devflow_temporal/delivery_readiness_gate_retry.py create mode 100644 runtime/tests/test_delivery_feature_readiness_handoff.py create mode 100644 runtime/tests/test_delivery_readiness_gate_retry.py diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index ebee2114..6616d626 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -181,6 +181,24 @@ create its reports before the dedicated browser gate. Nested output parents are created only inside an authenticated checkout. Existing unregistered outputs, tracked contents, protected paths and symlinked parents remain rejected. +A check stopped before launch by execution readiness records a versioned failure +class with its stage, candidate and immutable readiness-receipt digest. The +existing zero-repair gate control can admit at most two fresh readiness retries +for the unchanged candidate, preserving the old receipt, attempt and effect +ledger. Every retry probes readiness again and runs the original frozen gate; +its readiness receipt is never a gate pass. Other failed gates, pending product +findings, uncertain cleanup, changed source or unsettled effects refuse admission. +The initial route is qualified only for the fake fixture provider: native runs +refuse with `native_descendant_settlement_unknown` until a complete descendant +settlement contract can be authenticated. An exhausted retry allowance reports +`readiness_retry_limit_exhausted`. Neither blocker is a launch grant. A transient +unpublished stop with a recorded passing implementation is retained for an +authenticated same-candidate continuation; the automatic fresh-run path refuses +to replace it with another implementation. For a feature worker, the new +readiness path also requires an already authenticated successor assignment. The +current coordinator examines the stopped worker before reserving that assignment, +so an unreserved continuation remains blocked rather than bypassing ownership. + An older run stopped by that browser-output registration defect can use one `published_check_prelaunch_retry` with zero additional implementation turns. Admission requires its exact closed UNKNOWN checkpoint, passed local checks and diff --git a/runtime/src/devflow_temporal/delivery_activities.py b/runtime/src/devflow_temporal/delivery_activities.py index bca456e6..2f37ec2a 100644 --- a/runtime/src/devflow_temporal/delivery_activities.py +++ b/runtime/src/devflow_temporal/delivery_activities.py @@ -1221,12 +1221,13 @@ def _check_readiness(broker, request, attempt): workspace = broker.gate_checkout("verify", request["iteration"], request["candidate"]) requirements = describe_requirements( request["spec"], workspace, request["spec"].get("accepted_plan", "")) - generation = int(digest({"spec": request["spec"], "candidate": request["candidate"]})[:5], - 16) % 300000 - probe_attempt = 10 + generation * 3 + {100: 0, 101: 1, 102: 2}[attempt] + from .delivery_readiness_gate_retry import SLOTS, failure, probe_attempt + + stage = next(name for name, slot in SLOTS.items() if slot == attempt) + selected_attempt = probe_attempt(request["spec"], request["candidate"], attempt) readiness = prepare_and_probe(broker.store, request["spec"], request["candidate"], requirements, "plan", iteration=request["iteration"], - attempt=probe_attempt, workspace=workspace) + attempt=selected_attempt, workspace=workspace) if readiness.get("state") == "passed": verify_receipt(broker.store, request["spec"], request["candidate"], readiness, requirements) return readiness @@ -1237,7 +1238,8 @@ def _check_readiness(broker, request, attempt): "failure_kind": "readiness_preparation", "launched": False, "cleanup": cleanup, "candidate_id": request["candidate"]["id"], "source_unchanged": broker.candidate() == request["candidate"], - "reason": "execution readiness prevented the check", "readiness": readiness} + "reason": "execution readiness prevented the check", "readiness": readiness, + "failure_class": failure(readiness, stage, request["candidate"]["id"], cleanup)} @activity.defn(name="delivery_checks") diff --git a/runtime/src/devflow_temporal/delivery_automatic_retry.py b/runtime/src/devflow_temporal/delivery_automatic_retry.py index 00295fc5..bf5e01ee 100644 --- a/runtime/src/devflow_temporal/delivery_automatic_retry.py +++ b/runtime/src/devflow_temporal/delivery_automatic_retry.py @@ -554,6 +554,19 @@ def retry_once(store, *, stopped=lambda: False): 'SELECT issue_url FROM delivery_runs')) if count >= spec['policy']['max_attempts'] or unavailable(store, db, spec): continue + roles = result.get('roles') + if not isinstance(roles, list) or any(not isinstance(role, dict) for role in roles): + _record_skip(store, row, ValueError( + 'closed run lacks its complete implementation history')) + continue + if any(role.get('role') == 'implement' and role.get('status') == 'pass' + for role in roles): + # A fresh run would start implementation again and abandon the + # stopped candidate. It is not a same-candidate retry grant. + _record_skip(store, row, ValueError( + 'retained implementation requires an authenticated same-candidate ' + 'continuation; inspect the stopped recovery options')) + continue base = fresh_unpublished_base(store, spec) identity = digest({'run_id': spec['run_id'], 'request_digest': row['request_digest']})[:24] diff --git a/runtime/src/devflow_temporal/delivery_gate_retry.py b/runtime/src/devflow_temporal/delivery_gate_retry.py index a68a766e..21d948f5 100644 --- a/runtime/src/devflow_temporal/delivery_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_gate_retry.py @@ -74,8 +74,11 @@ def transient_stop(state): def prepublication_preparation_failed(state): """Recognize the controller's exact preparation stop, never a product finding.""" from .delivery_baseline_contract import preparation_failure + from .delivery_readiness_gate_retry import possible as readiness_possible precheck = state.get('checks', {}).get('prepublish', {}) + if readiness_possible(state) == 'prepublish' and precheck.get('state') == 'failed': + return True # Selection hint only; snapshot authenticates the receipt. prerequisite = preparation_failure(precheck) return (precheck.get('state') == 'failed' and prerequisite is not None and state.get('error') == f'environment preparation failed: {prerequisite}; ' @@ -547,6 +550,11 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= from .delivery_prerequisite_gate_recovery import possible as prerequisite_possible prerequisite_retry = kind == KIND and prerequisite_possible(spec, state) + from .delivery_readiness_gate_retry import possible as readiness_possible + + readiness_stage = readiness_possible(state) + readiness_retry = bool(readiness_stage and ( + (unpublished and readiness_stage == 'prepublish') or kind == KIND)) # A recorded gate verdict stands: a transport or controller stop after it is # never a fresh assessment of the same head. verdict = gate_failed(state.get('checks') or {}) @@ -587,7 +595,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= and previous.get('kind') == PREPUBLICATION_KIND and spec.get('gate_retry_generation') == 1) history = previous - ci_observations = transient_generations = 0 + ci_observations = transient_generations = readiness_generations = 0 published_after_recovery = bool(not unpublished and previous and previous.get('kind') in { PREPUBLICATION_KIND, 'pending_publication_retry', 'repair_continuation', 'published_head_validation'}) @@ -595,6 +603,8 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= 'finalized_checkpoint'): published_after_recovery = False while history: + if history.get('seal', {}).get('readiness_observation'): + readiness_generations += 1 if prerequisite_retry and history.get('seal', {}).get('ordinary_prerequisite_observation'): raise ValueError('ordinary prerequisite already received its one gate recovery') if browser_prelaunch and history.get('kind') == PRELAUNCH_KIND: @@ -613,7 +623,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= if transient_retry and transient_generations >= TRANSIENT_GATE_RETRIES: raise ValueError('this run already received its bounded transient gate retries') if ((renewed or published_after_recovery or prelaunch or report_retry or controller_only) - and spec['provider'] != 'fake'): + and not readiness_retry and spec['provider'] != 'fake'): from .delivery_native_preparation import native_identity if native_identity(spec)['runtime_payload_sha256'] in _consumed_payloads( spec, state, previous, include_implementation=controller_only): @@ -672,6 +682,31 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= browser_artifact_observation = None verification_observation = None prerequisite_observation = None + readiness_observation = None + if readiness_retry: + from .delivery_readiness_gate_retry import observe + + if state_checks != row_checks: + raise ValueError('readiness retry lost its exact original checks') + if spec.get('feature_worker'): + from .delivery_feature_execution import require_worker_handoff + + require_worker_handoff(store, spec) + readiness_observation = observe(spec, state) + latest = next((r for r in reversed(roles) if r.get('role') == 'implement'), None) + matches = [a for a in attempts if latest and a['role'] == 'implement' + and a['iteration'] == row['iteration'] + and a['session_id'] == latest.get('session_id')] + from .delivery_stopped_resume import _implementation_receipt + + if (not latest or latest.get('status') != 'pass' or latest.get('findings') + or latest.get('cleanup') != 'confirmed' or not latest.get('session_id') + or len(matches) != 1 or not _implementation_receipt(spec, latest, matches[0]) + or any((latest.get('candidate') or {}).get(key) != candidate.get(key) + for key in ('content_sha256', 'base_sha', 'environment_digest', + 'policy_digest'))): + raise ValueError('readiness retry lacks its exact passing implementation receipt') + failed_gate = True if prerequisite_retry: from .delivery_prerequisite_gate_recovery import observe @@ -770,7 +805,8 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= or (previous is not None and not ( renewed or published_after_recovery or prelaunch or report_retry or ci_only or transient_retry or attribution_retry - or verification_retry or browser_artifact_retry or prerequisite_retry)) + or verification_retry or browser_artifact_retry or prerequisite_retry + or readiness_retry)) or row['phase'] != 'blocked' or row['outcome'] != 'blocked' or row['execution_state'] != 'blocked' or row['cleanup'] != ('unknown' if prelaunch else 'confirmed') @@ -778,7 +814,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= or (prelaunch and state.get('cleanup') != 'unknown') or (not prelaunch and state.get('cleanup') != 'confirmed' and reconciled is None) or row['error'] != state.get('error') - or (not (transient_retry or transient_ci or masked_exhaustion) + or (not (transient_retry or transient_ci or masked_exhaustion or readiness_retry) and state.get('error') not in ({'repair limit exhausted', 'required CI did not confirm this PR head'} if report_retry @@ -842,7 +878,8 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= ancestor = ancestor.get('original_recovery') if admitted and (not (renewed or published_after_recovery or prelaunch or report_retry or ci_only or transient_retry or attribution_retry - or verification_retry or browser_artifact_retry or prerequisite_retry) + or verification_retry or browser_artifact_retry or prerequisite_retry + or readiness_retry) or (digest(prior_gate) != attribution_observation['prior_gate_digest'] if attribution_retry else prior_gate != ancestor)): raise ValueError('this run already received its bounded gate assessment retry') @@ -854,21 +891,47 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= store, spec, previous, candidate, publication, _inventory_profile) elif _inventory_profile is not None: raise ValueError('inventory physical custody belongs only to zero-model CI') + cleanup = (prerequisite_observation['resources'] if prerequisite_retry + else verification_observation['resources'] if verification_retry + else report_observation['resources'] if layout_report + else historical_teardown or prelaunch_observation or ( + _stopped_cleanup(spec, read_only=True) if _observation + else _stopped_cleanup(spec))) + if readiness_retry: + from .delivery_lifecycle import require_transition + from .delivery_readiness_gate_retry import RETRY_LIMIT + + # An observed native monitor does not prove its complete descendant set. + # The fake fixture has no native actor; real execution needs a separate + # settlement contract before this newly introduced route can launch work. + require_transition({ + 'operation': kind, 'owner': 'delivery_gate_retry', 'owner_proof': True, + 'normalized_lifecycle': 'stopped', + 'proof_digest': digest({'row': _sealed_row(row), 'cleanup': cleanup, + 'readiness': readiness_observation, + 'prior_readiness_retries': readiness_generations}), + 'comparisons': [], 'predicates': [ + {'code': 'native_descendant_settlement_unknown', + 'satisfied': True if spec['provider'] == 'fake' else None, + 'missing_or_unknown_fact': 'complete native descendant settlement', + 'next_action': 'obtain authenticated native descendant settlement'}, + {'code': 'readiness_retry_limit_exhausted', + 'satisfied': readiness_generations < RETRY_LIMIT, + 'missing_or_unknown_fact': 'remaining infrastructure retry allowance', + 'next_action': 'inspect retained failures and reconcile the environment'}, + ], + }, {}) result = {'row': _sealed_row(row), 'closed': closed, 'original_spec': spec, 'attempts': attempts, 'effects': effects, 'candidate': candidate, 'publication': publication, - 'cleanup': (prerequisite_observation['resources'] if prerequisite_retry - else verification_observation['resources'] if verification_retry - else report_observation['resources'] if layout_report - else historical_teardown or prelaunch_observation or ( - _stopped_cleanup(spec, read_only=True) if _observation - else _stopped_cleanup(spec))), + 'cleanup': cleanup, 'work_binding': binding, 'previous_digest': digest(previous) if previous else None, 'prior_gate_digest': digest(prior_gate) if prior_gate else None, **({'local_download_retry': True} if local_download_retry else {}), **({'ordinary_prerequisite_observation': prerequisite_observation} if prerequisite_observation else {}), + **({'readiness_observation': readiness_observation} if readiness_observation else {}), **({'report_observation': report_observation} if report_observation else {}), **({'verification_observation': verification_observation} if verification_observation else {}), @@ -881,13 +944,14 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= if attribution_retry or layout_report or verification_retry else {}), **({'browser_prelaunch_observation': browser_observation} if browser_observation else {}), - 'stage': 'prerequisite' if prerequisite_retry + 'stage': 'readiness' if readiness_retry else 'prerequisite' if prerequisite_retry else 'transient' if transient_retry or verification_retry else 'report' if report_retry else 'ci' if ci_only else 'published' if published_after_recovery or prelaunch or controller_only or attribution_retry or browser_artifact_retry else None, - 'generation': (transient_generations + 1 if transient_retry + 'generation': (readiness_generations + 1 if readiness_retry + else transient_generations + 1 if transient_retry else ci_observations + 1 if ci_only else (2 if previous else 1) if attribution_retry or browser_artifact_retry @@ -940,6 +1004,8 @@ def admit(store, run_id, payload, *, preflight=False): return json.loads(prior['response_json']) seal = snapshot(store, run_id, kind, _observation=preflight) previous = _retained_recovery(store, run_id, seal) + if seal.get('readiness_observation') and 'verification_test_paths' in payload: + raise ValueError('readiness retry must retain the original frozen test selectors') if 'verification_test_paths' in payload: from .delivery_plan_checks import planned_checks @@ -974,7 +1040,7 @@ def admit(store, run_id, payload, *, preflight=False): ordering = (seal.get('browser_artifact_observation') or {}).get( 'cause') == 'browser_artifact_assessment_order' if (preflight_generation or seal.get('preparation_observation') - or layout or verification or ordering) \ + or seal.get('readiness_observation') or layout or verification or ordering) \ and spec.get('feature_worker'): from .delivery_feature_execution import require_worker_handoff @@ -1022,7 +1088,8 @@ def admit(store, run_id, payload, *, preflight=False): execution['implementation_role_home_generation'] = implementation_generation( spec, previous) execution['role_home_generation'] = ( - f'report-retry-{generation}' if seal['stage'] == 'report' + f'readiness-retry-{generation}' if seal['stage'] == 'readiness' + else f'report-retry-{generation}' if seal['stage'] == 'report' else f'transient-gate-retry-{generation}' if seal['stage'] == 'transient' else f'gate-retry-{generation}') execution['gate_retry_generation'] = generation @@ -1071,7 +1138,9 @@ def admit(store, run_id, payload, *, preflight=False): else {**seal['publication'], 'candidate': after})} _immutable(root / 'admission.json', recovery) preserve_resources(root, spec) - workflow_id = (f'delivery-{run_id}-ci-retry-{generation}' if kind == CI_KIND else + workflow_id = (f'delivery-{run_id}-readiness-gates-retry-{generation}' + if seal['stage'] == 'readiness' else + f'delivery-{run_id}-ci-retry-{generation}' if kind == CI_KIND else f'delivery-{run_id}-report-gates-retry-{generation}' if seal['stage'] == 'report' else f'delivery-{run_id}-transient-gates-retry-{generation}' @@ -1093,19 +1162,19 @@ def admit(store, run_id, payload, *, preflight=False): if canonical_json(_sealed_row(row)) != canonical_json(seal['row']): raise ValueError('gate retry admission lost its stopped projection') if (verification or ordering or seal.get('ordinary_prerequisite_observation') - or seal.get('ordinary_ci_inventory')): + or seal.get('ordinary_ci_inventory') or seal.get('readiness_observation')): if store.state.claim_for(db, spec['work_id']) is not None: - raise ValueError('listener verification admission lost its released claim') + raise ValueError('gate retry admission lost its released claim') for table, key, expected in [('delivery_attempts', 'job_key', seal['attempts']), ('delivery_effects', 'effect_key', seal['effects'])]: records = [dict(item) for item in db.execute( f'SELECT * FROM {table} WHERE run_id=? ORDER BY {key}', (run_id,))] if records != expected: - raise ValueError('listener verification admission lost its full ledger') + raise ValueError('gate retry admission lost its full ledger') if work_binding(store, spec, db) != seal['work_binding']: raise ValueError('gate retry admission lost issue ownership') if ((preflight_generation or seal.get('preparation_observation') - or layout or verification or ordering) + or seal.get('readiness_observation') or layout or verification or ordering) and spec.get('feature_worker')): require_worker_handoff(store, spec) if preparation_owner is not None: @@ -1171,6 +1240,8 @@ def gate_namespace(generation, stage=None, *, preflight_generation=0, prepared_h suffix = f'-{preflight_generation}' if preflight_generation > 1 else '' return gate_namespace(generation, stage) + '/preflight-continuation' + suffix if stage is not None: + if stage == 'readiness' and type(generation) is int and generation in (1, 2): + return f'readiness-gates-admission-{generation}' if stage == 'prerequisite' and type(generation) is int and generation == 1: return 'ordinary-prerequisite-gates-admission' if stage == 'preparation' and type(generation) is int and generation in (1, 2): @@ -1226,6 +1297,17 @@ def readback(store, spec, recovery, *, resource_spec=None): if resource_spec is not None else DeliveryBroker(store, spec)) if effective != spec or broker.candidate() != recovery['candidate']: raise ValueError('gate retry source or execution authority changed') + if expected := recovery['seal'].get('readiness_observation'): + from .delivery_readiness_gate_retry import observe + + seal = recovery['seal'] + _, attempts, effects, claim = _rows(store, spec['run_id']) + closed = store._completed_temporal_result( + spec['run_id'], workflow_id=seal['closed']['workflow_id']) + if (closed != seal['closed'] or attempts != seal['attempts'] or effects != seal['effects'] + or claim is None or claim['owner'] != f"external:devflow:{spec['run_id']}" + or observe(seal['original_spec'], recovery['state']) != expected): + raise ValueError('readiness retry original evidence or ledger changed') if recovery['seal'].get('ordinary_ci_inventory'): from .delivery_ci_inventory import validate diff --git a/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py b/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py new file mode 100644 index 00000000..47cd5ca0 --- /dev/null +++ b/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py @@ -0,0 +1,127 @@ +"""Authenticate an unlaunched gate readiness refusal for bounded reassessment. + +The immutable refusal is historical evidence, not a readiness pass or a launch +permission. The admitted continuation must probe again before any required gate. +""" +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +from .contracts import digest +from .delivery_runtime_location import private_bytes + +RETRY_LIMIT = 2 +SLOTS = {"prepublish": 100, "local": 101, "browser_qa": 102} + + +def probe_attempt(spec, candidate, slot): + """Reserve disjoint readiness evidence names for the two retry generations.""" + if slot not in SLOTS.values(): + raise ValueError("unsupported gate readiness slot") + if spec.get("gate_retry_stage") == "readiness": + generation = spec.get("gate_retry_generation") + if type(generation) is not int or not 1 <= generation <= RETRY_LIMIT: + raise ValueError("readiness retry generation exceeds its finite bound") + # Existing hash-derived attempts are in [10, 900009]. + return 900010 + (generation - 1) * 3 + slot - 100 + generation = int(digest({"spec": spec, "candidate": candidate})[:5], 16) % 300000 + return 10 + generation * 3 + slot - 100 + + +def failure(readiness, stage, candidate_id, cleanup): + state = readiness.get("state") + classification = ("environment_prelaunch" if state == "failed" and cleanup == "confirmed" + else "cancelled" if state == "cancelled" else "unknown") + return {"version": 1, "class": classification, "source": "execution_readiness", + "stage": stage, "candidate_id": candidate_id, + "receipt_sha256": readiness.get("receipt_sha256"), "launched": False} + + +def possible(state): + """Selection hint only. `observe` authenticates the result and its receipt.""" + checks = state.get("checks") or {} + if not isinstance(checks, dict): + return None + found = [stage for stage in SLOTS if isinstance(checks.get(stage), dict) + and checks[stage].get("failure_kind") == "readiness_preparation" + and checks[stage].get("launched") is False] + return found[0] if len(found) == 1 else None + + +def observe(spec, state): + """Bind a completed controller result to its original private failure receipt.""" + stage = possible(state) + checks, candidate = state.get("checks") or {}, state.get("candidate") or {} + roles = state.get("roles") + if (not isinstance(roles, list) or any(not isinstance(role, dict) for role in roles)): + raise ValueError("readiness retry lacks its complete role history") + if (spec.get("readiness_version") != 1 or stage is None + or type(state.get("iteration")) is not int or state["iteration"] < 0 + or not isinstance(candidate, dict) or not candidate.get("id")): + raise ValueError("readiness retry requires an exact unlaunched gate refusal") + expected_error = {"prepublish": "execution readiness prevented prepublication", + "local": "execution readiness prevented local checks", + "browser_qa": "execution readiness prevented browser QA"}[stage] + if state.get("error") != expected_error: + raise ValueError("readiness retry lost its exact controller stop") + result = checks[stage] + receipt = result.get("readiness") + if (not isinstance(receipt, dict) or result.get("state") != "failed" + or result.get("cleanup") != "confirmed" or result.get("source_unchanged") is not True + or result.get("candidate_id") != candidate["id"] + or checks.get("readiness") != receipt or receipt.get("version") != 1 + or receipt.get("stage") != "plan" or receipt.get("state") != "failed" + or receipt.get("iteration") != state["iteration"] + or receipt.get("attempt") != probe_attempt(spec, candidate, SLOTS[stage])): + raise ValueError("readiness retry lacks a confirmed candidate-bound refusal") + provenance = failure(receipt, stage, candidate["id"], "confirmed") + if result.get("failure_class") not in (None, provenance): + raise ValueError("readiness failure provenance changed") + # A failed or unassessed product gate remains authoritative. Never replace + # pending findings with a same-candidate retry. + for name in ("prepublish", "local", "browser_qa", "review", "qa", "ci"): + item = checks.get(name) + if name != stage and isinstance(item, dict) and item.get("state") != "passed": + raise ValueError("readiness retry cannot replace another gate verdict") + if any(role.get("iteration") == state["iteration"] + and role.get("role") in {"review", "verify"} + and (role.get("status") != "pass" or role.get("findings")) + for role in roles): + raise ValueError("readiness retry cannot replace an independent product finding") + pending = state.get("repair_findings") or {} + if not isinstance(pending, dict): + raise ValueError("readiness retry lacks its pending finding history") + if pending.get("iteration") == state["iteration"] and pending.get("findings"): + raise ValueError("readiness retry cannot discard pending product findings") + subject = receipt.get("binding") or {} + observations = receipt.get("observations") + if (not isinstance(subject, dict) or subject.get("version") != 1 + or subject.get("spec_sha256") != digest(spec) or subject.get("candidate") != candidate + or not isinstance(subject.get("requirements_sha256"), str) + or not re.fullmatch(r"[0-9a-f]{64}", subject["requirements_sha256"]) + or type(subject.get("resource_generation")) is not int + or subject["resource_generation"] < 0 or not isinstance(subject.get("resources"), dict) + or not isinstance(observations, dict) or not observations + or not any(isinstance(item, dict) and item.get("state") == "failed" + for item in observations.values()) + or any(not isinstance(item, dict) or not isinstance(item.get("state"), str) + or item.get("state") not in {"passed", "failed"} + or item.get("cleanup") != "confirmed" or item.get("candidate") != candidate + or item.get("requirement_sha256") != subject["requirements_sha256"] + for item in observations.values())): + raise ValueError("readiness retry lacks complete confirmed failure observations") + path = (Path(spec["state_dir"]) / "readiness" / "receipts" / "plan" + / str(state["iteration"]) / f"{receipt['attempt']}.json") + if receipt.get("receipt") != str(path): + raise ValueError("readiness failure receipt left its owned run") + raw = private_bytes(path, maximum=8 * 1024 * 1024) + if (hashlib.sha256(raw).hexdigest() != receipt.get("receipt_sha256") + or json.loads(raw) != {key: value for key, value in receipt.items() + if key not in {"receipt", "receipt_sha256"}}): + raise ValueError("readiness failure receipt changed") + return {"stage": stage, "candidate_id": candidate["id"], "attempt": receipt["attempt"], + "receipt": str(path), "receipt_sha256": receipt["receipt_sha256"], + "requirements_sha256": subject["requirements_sha256"], "failure_class": provenance} diff --git a/runtime/src/devflow_temporal/delivery_workflow.py b/runtime/src/devflow_temporal/delivery_workflow.py index 3ddb186e..80f90ed3 100644 --- a/runtime/src/devflow_temporal/delivery_workflow.py +++ b/runtime/src/devflow_temporal/delivery_workflow.py @@ -2664,6 +2664,16 @@ async def _run_iterations( return await self._stop(spec, 'technical prepublication checks failed: ' + type(exc).__name__, cause=exc) self.state['checks']['prepublish'] = checked + if (spec.get('gate_retry_stage') == 'readiness' + and spec.get('readiness_version') == 1 + and checked.get('readiness')): + self.state['checks']['readiness'] = checked['readiness'] + if (checked.get('failure_kind') == 'readiness_preparation' + and checked.get('launched') is False): + if checked.get('cleanup') == 'unknown': + self.state['cleanup'] = 'unknown' + return await self._stop( + spec, 'execution readiness prevented prepublication') if checked.get('state') != 'passed' or checked.get('cleanup') == 'unknown': return await self._stop(spec, 'technical prepublication checks failed') elif recovery is not None and iteration == start_iteration: diff --git a/runtime/tests/test_delivery_automatic_retry.py b/runtime/tests/test_delivery_automatic_retry.py index 08bf9ff7..b9b67d66 100644 --- a/runtime/tests/test_delivery_automatic_retry.py +++ b/runtime/tests/test_delivery_automatic_retry.py @@ -30,7 +30,7 @@ def stopped(service, monkeypatch): 'closed_at': '2026-10-07T00:00:00+00:00', 'request_digest': spec['request_digest'], 'recovery_digest': None, 'result': {'run_id': request['run_id'], 'outcome': 'blocked', 'phase': 'blocked', 'execution_state': 'blocked', - 'cleanup': 'confirmed', 'checks': checks}} + 'cleanup': 'confirmed', 'checks': checks, 'roles': []}} monkeypatch.setattr(DeliveryStore, '_completed_temporal_result', lambda *_a, **_k: closed) monkeypatch.setattr(retry, 'observe_finalized_resources', lambda *_a, **_k: {'finalization_sha256': 'fixture-finalization'}) @@ -150,3 +150,16 @@ def test_automatic_generations_spend_one_original_total_ceiling(service, monkeyp assert retry.retry_once(store) == [] with store._connect() as db: assert db.execute('SELECT COUNT(*) FROM delivery_runs').fetchone()[0] == 3 + +def test_transient_stop_after_implementation_retains_candidate_instead_of_fresh_run( + service, monkeypatch, +): + store, request, closed = stopped(service, monkeypatch) + closed['result'].update(candidate={'id': 'retained-candidate'}, iteration=0, roles=[{ + 'role': 'implement', 'iteration': 0, 'status': 'pass', 'session_id': 'retained-session'}]) + assert retry.retry_once(store) == [] + with store._connect() as db: + assert db.execute('SELECT COUNT(*) FROM delivery_runs').fetchone()[0] == 1 + events = [json.loads(row[0]) for row in db.execute( + "SELECT payload_json FROM delivery_events WHERE type='automatic_retry_skipped'")] + assert any('same-candidate continuation' in event['reason'] for event in events) diff --git a/runtime/tests/test_delivery_feature_readiness_handoff.py b/runtime/tests/test_delivery_feature_readiness_handoff.py new file mode 100644 index 00000000..37a300aa --- /dev/null +++ b/runtime/tests/test_delivery_feature_readiness_handoff.py @@ -0,0 +1,195 @@ +"""Observe the feature-worker readiness handoff boundary without a provider.""" + +import json +from copy import deepcopy + +import pytest +from test_delivery_feature_execution import feature_service +from test_delivery_store import service as service + +from devflow_temporal import delivery_feature_activities as activities +from devflow_temporal.contracts import canonical_json, digest +from devflow_temporal.delivery_activities import _check_readiness +from devflow_temporal.delivery_broker import DeliveryBroker +from devflow_temporal.delivery_config import DeliveryConfig +from devflow_temporal.delivery_execution_registry import OwnershipConflict +from devflow_temporal.delivery_feature_execution import ( + register_worker, + registry, + worker_key, + worker_spec, +) +from devflow_temporal.delivery_github_contract import ordered_chunks +from devflow_temporal.delivery_resources import RunResources + + +def test_feature_readiness_requires_reserved_successor_and_preserves_budget(service, monkeypatch): + store, request, _ = feature_service(service, monkeypatch) + original_admit = DeliveryConfig.admit + + def admit(config, *args, **kwargs): + spec = original_admit(config, *args, **kwargs) + spec["readiness_version"] = 1 + spec["policy"]["readiness"] = { + "version": 1, + "generation": 0, + "browser_engines": [], + "temporal_checks": [], + } + spec["policy_digest"] = digest(spec["policy"]) + return spec + + monkeypatch.setattr(DeliveryConfig, "admit", admit) + store.submit(request) + parent = store.effective_spec(request["run_id"]) + chunk = ordered_chunks(json.loads(request["accepted_plan"]))[0] + child = worker_spec( + parent, + chunk, + {"url": "https://github.com/example/fixture/issues/10"}, + kind="chunk", + base_sha=parent["base_sha"], + base_branch="main", + ) + register_worker(store, parent, child) + broker = DeliveryBroker(store, child) + broker.prepare() + resources = RunResources(child) + stage = broker.checkout.with_name("staged-feature-candidate") + broker.checkout.rename(stage) + resources.register(broker.checkout, "checkout") + stage.rename(broker.checkout) + resources.created(broker.checkout) + (broker.checkout / "README.md").write_text("Retained feature candidate\n") + candidate = broker.candidate() + monkeypatch.setattr( + "devflow_temporal.delivery_preparation.verify_prepared_spec", lambda *_: None + ) + monkeypatch.setattr( + "devflow_temporal.delivery_configured_resources.implementation_prerequisites", lambda *_: [] + ) + monkeypatch.setattr( + "devflow_temporal.delivery_plan_checks.planned_checks", lambda *_a, **_k: [] + ) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda *_a, **_k: { + "state": "failed", + "results": [{"id": "missing", "passed": False, "cleanup": "confirmed"}], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + result = _check_readiness(broker, {"spec": child, "candidate": candidate, "iteration": 0}, 100) + implementation = { + "role": "implement", + "iteration": 0, + "status": "pass", + "session_id": "original-implementation", + "cleanup": "confirmed", + "candidate": candidate, + } + checks = {"prepublish": result, "readiness": result["readiness"]} + state = { + "run_id": child["run_id"], + "revision": 9, + "iteration": 0, + "phase": "blocked", + "execution_state": "blocked", + "outcome": "blocked", + "cleanup": "confirmed", + "candidate": candidate, + "pull_request": None, + "roles": [implementation], + "checks": checks, + "error": "execution readiness prevented prepublication", + } + store.project( + child["run_id"], + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=state["error"], + outcome="blocked", + cleanup="confirmed", + candidate=candidate, + checks=checks, + pull_request=None, + error=state["error"], + iteration=0, + protocol_revision=9, + ) + shared = registry(parent) + token = parent["feature_delivery"]["owner"] + shared.finish_worker(token, worker_key(child["run_id"], token), {"cleanup": "confirmed"}) + before = deepcopy(shared.budget(token["issue_id"])) + shared.stop(token, "stopped-for-preparation", {"cleanup": "confirmed"}) + with store._connect() as db: + store.state.release_work(db, parent["work_id"], "external:devflow:" + parent["run_id"]) + store.submit( + { + **request, + "run_id": "run-successor", + "command_id": "continue-preparation", + "feature_predecessor": token, + } + ) + successor = store.effective_spec("run-successor") + with store._connect() as db: + db.execute( + "INSERT INTO delivery_attempts(job_key,run_id,role,iteration,candidate_id," + "state,session_id,result_json,cleanup) VALUES " + "('implementation',?,'implement',0,?,'finished',?,?,'confirmed')", + ( + child["run_id"], + candidate["id"], + implementation["session_id"], + canonical_json( + { + k: v + for k, v in implementation.items() + if k not in {"role", "iteration", "candidate"} + } + ), + ), + ) + store.state.release_work(db, child["work_id"], "external:devflow:" + child["run_id"]) + row = dict( + db.execute("SELECT * FROM delivery_runs WHERE run_id=?", (child["run_id"],)).fetchone() + ) + closed = { + "workflow_id": row["workflow_id"], + "execution_run_id": "closed-worker", + "request_digest": child["request_digest"], + "recovery_digest": None, + "result": state, + } + monkeypatch.setattr(store, "_completed_temporal_result", lambda *_a, **_k: closed) + monkeypatch.setattr(DeliveryBroker, "_existing_pr", lambda *_a, **_k: None) + monkeypatch.setattr( + "devflow_temporal.delivery_gate_retry._stopped_cleanup", lambda *_a, **_k: {} + ) + with pytest.raises(OwnershipConflict, match="active execution assignment"): + activities.resume_worker(store, successor, child, row) + assert shared.budget(token["issue_id"]) == before + with store._connect() as db: + assert not db.execute("SELECT 1 FROM delivery_gate_admissions").fetchone() + next_token = successor["feature_delivery"]["owner"] + shared.reserve_worker( + next_token, + worker_key(child["run_id"], next_token), + child["feature_worker"]["workstream_id"], + ) + continued = activities.resume_worker(store, successor, child, row) + assert continued["resumed"] and continued["spec"]["run_id"] == child["run_id"] + assert shared.budget(token["issue_id"]) == before + with store._connect() as db: + recovery = json.loads( + db.execute( + "SELECT recovery_json FROM delivery_runs WHERE run_id=?", (child["run_id"],) + ).fetchone()[0] + ) + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert recovery["command"]["additional_iterations"] == 0 + assert recovery["state"]["checks"]["prepublish"]["readiness"] == result["readiness"] diff --git a/runtime/tests/test_delivery_readiness_gate_retry.py b/runtime/tests/test_delivery_readiness_gate_retry.py new file mode 100644 index 00000000..d56fbbd6 --- /dev/null +++ b/runtime/tests/test_delivery_readiness_gate_retry.py @@ -0,0 +1,739 @@ +"""Public same-candidate retry of a real, retained readiness refusal. + +The Temporal closure, implementation and failed dependency are controlled +fixtures. Readiness receipt creation, Git/SQLite ownership and HTTP admission +are real; no provider or production installation is used. +""" + +from __future__ import annotations + +import json +from contextlib import asynccontextmanager +from copy import deepcopy +from pathlib import Path + +import httpx +import pytest +from test_delivery_gate_retry import unpublished as unpublished +from test_delivery_metadata_recovery import published as published +from test_delivery_store import _git +from test_delivery_store import service as base_service + +from devflow_temporal.contracts import canonical_json, digest +from devflow_temporal.delivery_activities import _check_readiness +from devflow_temporal.delivery_api import create_app +from devflow_temporal.delivery_broker import DeliveryBroker +from devflow_temporal.delivery_config import DeliveryConfig +from devflow_temporal.delivery_gate_retry import KIND, readback +from devflow_temporal.delivery_resources import RunResources +from devflow_temporal.delivery_store import DeliveryStore + + +@pytest.fixture +def service(tmp_path): + store, request = base_service.__wrapped__(tmp_path) + source = Path(store.config.raw["repositories"]["fixture"]["source_path"]) + (source / "test_retry_gate.py").write_text( + "import unittest\n" + "from pathlib import Path\n" + "class Gate(unittest.TestCase):\n" + " def test_candidate(self):\n" + " self.assertIn('Owned unpublished candidate', Path('README.md').read_text())\n" + " def test_absent(self):\n" + " self.assertIn('an absent requirement', Path('README.md').read_text())\n" + ) + _git(source, "add", "test_retry_gate.py") + _git(source, "commit", "-qm", "Add fixture check") + config = deepcopy(store.config.raw) + config["repositories"]["fixture"]["expected_base_sha"] = _git(source, "rev-parse", "HEAD") + store.config.path.write_text(json.dumps(config)) + return DeliveryStore(DeliveryConfig.load(store.config.path)), request + + +@pytest.fixture +def stopped_readiness(unpublished, monkeypatch, request): + store, _, state, command = unpublished + spec = deepcopy(store.spec("run-1")) + spec["readiness_version"] = 1 + spec["policy"]["prepublish_checks"] = [ + { + "id": "diff", + "argv": ["git", "diff", "--check"], + "cwd": ".", + "kind": "check", + "timeout_seconds": 30, + } + ] + selector = ( + "test_retry_gate.Gate.test_absent" + if getattr(request, "param", "") == "assertion" + else "test_retry_gate.Gate.test_candidate" + ) + spec["policy"]["checks"] = [ + { + "id": "assert-readme", + "argv": ["python3", "-m", "unittest", selector], + "cwd": ".", + "kind": "check", + "timeout_seconds": 30, + } + ] + spec["policy"]["readiness"] = { + "version": 1, + "generation": 0, + "browser_engines": [], + "temporal_checks": [], + } + browser_missing = getattr(request, "param", "") == "missing-browser" + if browser_missing: + spec["policy"]["browser_qa"] = {"argv": ["playwright", "test"], "ports": {}} + spec["policy"]["readiness"]["browser_engines"] = ["chromium"] + monkeypatch.setattr(RunResources, "browser_scratch", lambda self: Path(spec["state_dir"])) + monkeypatch.setattr( + "devflow_temporal.delivery_sandbox.prepare_browser_qa", + lambda *_args, **_kwargs: (None, {"PATH": str(Path(spec["state_dir"]) / "no-node")}), + ) + spec["policy_digest"] = digest(spec["policy"]) + with store._connect() as db: + db.execute( + "UPDATE delivery_runs SET request_json=? WHERE run_id=?", + (canonical_json(spec), "run-1"), + ) + broker = DeliveryBroker(store, spec) + resources = RunResources(spec) + staged = broker.checkout.with_name("staged-candidate") + broker.checkout.rename(staged) + resources.register(broker.checkout, "checkout") + staged.rename(broker.checkout) + resources.created(broker.checkout) + candidate = broker.candidate() + state["candidate"] = candidate + state["roles"][0]["candidate"] = candidate + with store._connect() as db: + role_result = { + k: v + for k, v in state["roles"][0].items() + if k not in {"role", "iteration", "candidate"} + } + db.execute( + "UPDATE delivery_attempts SET result_json=? WHERE job_key='original'", + (canonical_json(role_result),), + ) + command["expected_candidate_id"] = candidate["id"] + monkeypatch.setattr( + "devflow_temporal.delivery_preparation.verify_prepared_spec", lambda *_: None + ) + monkeypatch.setattr( + "devflow_temporal.delivery_configured_resources.implementation_prerequisites", lambda *_: [] + ) + monkeypatch.setattr( + "devflow_temporal.delivery_plan_checks.planned_checks", lambda *_a, **_k: [] + ) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, *_a, **_k: { + "state": "failed", + "results": [{"id": "dependency", "passed": False, "cleanup": "confirmed"}], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + + def failure(execution, current): + broker = DeliveryBroker(store, execution) + result = _check_readiness( + broker, {"spec": execution, "candidate": current["candidate"], "iteration": 0}, 100 + ) + assert result["failure_class"]["class"] == "environment_prelaunch" + assert result["readiness"]["state"] == "failed" + current["checks"] = {"prepublish": result, "readiness": result["readiness"]} + current["error"] = "execution readiness prevented prepublication" + return result + + original = failure(spec, state) + store.project( + "run-1", + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=state["error"], + candidate=candidate, + checks=state["checks"], + iteration=0, + protocol_revision=9, + outcome="blocked", + cleanup="confirmed", + error=state["error"], + ) + closed = { + "workflow_id": "delivery-run-1", + "execution_run_id": "closed-original", + "request_digest": spec["request_digest"], + "recovery_digest": None, + "result": state, + } + monkeypatch.setattr(store, "_completed_temporal_result", lambda *_a, **_k: closed) + monkeypatch.setattr(DeliveryStore, "_completed_temporal_result", lambda *_a, **_k: closed) + return store, broker, state, command, original, closed, failure + + +@asynccontextmanager +async def client_for(store): + app = create_app(store.config.path) + host = store.config.dashboard_url + client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=app, client=("127.0.0.1", 8001)), base_url=host + ) + async with client: + headers = { + "Origin": host, + "X-Devflow-CSRF": (await client.get("/api/session")).json()["csrf_token"], + } + yield client, headers + + +@pytest.mark.asyncio +async def test_public_retry_preserves_receipts_and_bounds_repeated_failures(stopped_readiness): + store, broker, state, command, original, closed, failure = stopped_readiness + candidate = deepcopy(state["candidate"]) + retained = Path(original["readiness"]["receipt"]).read_bytes() + with store._connect() as db: + initial_attempts = [dict(x) for x in db.execute("SELECT * FROM delivery_attempts")] + initial_effects = [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] + attempts = [original["readiness"]["attempt"]] + async with client_for(store) as (client, headers): + for generation in (1, 2): + preflight = await client.post( + "/api/runs/run-1/repair-admission-preflight", json=command + ) + assert preflight.status_code == 200, preflight.text + result = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert result.status_code == 200, result.text + admitted = result.json() + assert admitted["implementation_authority"] is False + assert ( + admitted["additional_iterations"] == 0 + and admitted["candidate_id"] == candidate["id"] + ) + assert ( + await client.post("/api/runs/run-1/continue-repair", json=command, headers=headers) + ).json() == admitted + execution = store.effective_spec("run-1") + with store._connect() as db: + recovery = json.loads( + db.execute( + "SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'" + ).fetchone()[0] + ) + assert [ + dict(x) for x in db.execute("SELECT * FROM delivery_attempts") + ] == initial_attempts + assert [ + dict(x) for x in db.execute("SELECT * FROM delivery_effects") + ] == initial_effects + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert execution["gate_retry_generation"] == generation + assert execution["gate_retry_stage"] == "readiness" + assert recovery["state"]["checks"]["prepublish"]["readiness"]["state"] == "failed" + assert readback(store, execution, recovery) is None + if generation == 1: + old_path = Path(original["readiness"]["receipt"]) + old_path.write_bytes(retained + b" ") + with pytest.raises(ValueError, match="receipt changed"): + readback(store, execution, recovery) + old_path.write_bytes(retained) + assert broker.candidate() == candidate + current = deepcopy(recovery["state"]) + current.update( + revision=9 + generation, + phase="blocked", + execution_state="blocked", + outcome="blocked", + cleanup="confirmed", + candidate=recovery["candidate"], + ) + failed_again = failure(execution, current) + attempts.append(failed_again["readiness"]["attempt"]) + store.project( + "run-1", + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=current["error"], + candidate=current["candidate"], + checks=current["checks"], + iteration=0, + protocol_revision=current["revision"], + outcome="blocked", + cleanup="confirmed", + error=current["error"], + ) + with store._connect() as db: + store.state.release_work(db, execution["work_id"], "external:devflow:run-1") + closed.update( + workflow_id=admitted["workflow_id"], + execution_run_id=f"closed-{generation}", + recovery_digest=digest(recovery), + result=current, + ) + command = { + **command, + "command_id": f"retry-readiness-{generation + 1}", + "expected_revision": current["revision"], + } + refusal = await client.post("/api/runs/run-1/repair-admission-preflight", json=command) + assert refusal.status_code == 409 + assert "readiness_retry_limit_exhausted" in refusal.text + assert len(set(attempts)) == 3 + assert Path(original["readiness"]["receipt"]).read_bytes() == retained + with store._connect() as db: + assert [dict(x) for x in db.execute("SELECT * FROM delivery_attempts")] == initial_attempts + assert [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] == initial_effects + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drift", ["receipt", "source", "effect", "product", "cleanup"]) +async def test_public_retry_refuses_changed_or_unsettled_evidence(stopped_readiness, drift): + store, broker, state, command, original, _, _ = stopped_readiness + if drift == "receipt": + path = Path(original["readiness"]["receipt"]) + path.write_bytes(path.read_bytes() + b" ") + elif drift == "source": + (broker.checkout / "README.md").write_text("Changed candidate\n") + elif drift == "effect": + with store._connect() as db: + db.execute( + "INSERT INTO delivery_effects VALUES ('pending','run-1','publish'," + "'{}','pending',NULL,'now')" + ) + elif drift == "product": + state["roles"].append( + { + "role": "review", + "iteration": 0, + "status": "findings", + "findings": ["Real product defect"], + "cleanup": "confirmed", + } + ) + else: + state["cleanup"] = "unknown" + with store._connect() as db: + db.execute("UPDATE delivery_runs SET cleanup='unknown' WHERE run_id='run-1'") + async with client_for(store) as (client, _): + response = await client.post("/api/runs/run-1/repair-admission-preflight", json=command) + assert response.status_code == 409, response.text + with store._connect() as db: + assert not db.execute("SELECT 1 FROM delivery_gate_admissions").fetchone() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stopped_readiness", ["passing", "assertion"], indirect=True) +async def test_retry_workflow_runs_the_frozen_gates_without_implementation( + stopped_readiness, monkeypatch +): + import asyncio + + from temporal_test_server import local_temporal + from temporalio import activity + from temporalio.worker import Worker + + from devflow_temporal.delivery_activities import ( + delivery_checks, + delivery_gates_readback, + delivery_precheck, + delivery_project, + ) + from devflow_temporal.delivery_workflow import DeliveryWorkflow + + store, broker, state, command, original, _, _ = stopped_readiness + old_receipt = Path(original["readiness"]["receipt"]).read_bytes() + async with client_for(store) as (client, headers): + response = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert response.status_code == 200, response.text + admitted = response.json() + spec = store.effective_spec("run-1") + with store._connect() as db: + recovery = json.loads( + db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0] + ) + store.mark_start("run-1", accepted=True) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, _i, candidate, **_k: { + "state": "passed", + "results": [], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + monkeypatch.setattr( + "devflow_temporal.delivery_readiness.probe_tools", + lambda _broker, requirements, candidate, **_kw: { + "state": "passed", + "cleanup": "confirmed", + "component": "tools", + "candidate": candidate, + "requirement_sha256": digest(requirements), + "items": [], + }, + ) + monkeypatch.setattr("devflow_temporal.delivery_readiness_verify.tools", lambda *_a: None) + actual_gate_checkout = DeliveryBroker.gate_checkout + + def gate_checkout(self, role, iteration, candidate): + path = actual_gate_checkout(self, role, iteration, candidate) + resources = self.run_resources() + with resources.locked(read_only=True) as manifest: + registered = str(path) in manifest["roots"] + if not registered: + staged = path.with_name(path.name + "-staged") + path.rename(staged) + resources.register(path, "gate") + staged.rename(path) + resources.created(path, head=candidate["head"]) + return path + + monkeypatch.setattr(DeliveryBroker, "gate_checkout", gate_checkout) + calls = [] + + @activity.defn(name="delivery_publish") + async def publish(body): + calls.append("publish") + assert body["candidate"]["id"] == state["candidate"]["id"] + _git(broker.checkout, "add", "README.md") + _git(broker.checkout, "commit", "-qm", "Publish fixture") + current = DeliveryBroker(store, body["spec"]).candidate() + return { + "number": 7, + "url": "https://example.invalid/pull/7", + "state": "OPEN", + "head": current["head"], + "candidate": current, + } + + @activity.defn(name="delivery_role") + async def role(body): + calls.append(body["role"]) + assert body["role"] in {"review", "verify"} + return { + "role": body["role"], + "iteration": body["iteration"], + "status": "pass", + "session_id": "independent-" + body["role"], + "candidate": body["candidate"], + "cleanup": "confirmed", + "findings": [], + } + + @activity.defn(name="delivery_tracker_start") + async def start(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_tracker") + async def tracker(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_ci") + async def ci(_body): + calls.append("ci") + return {"state": "passed"} + + async with local_temporal() as env: + async with Worker( + env.client, + task_queue="issue-243-gates", + workflows=[DeliveryWorkflow], + activities=[ + delivery_gates_readback, + delivery_project, + delivery_precheck, + delivery_checks, + publish, + role, + start, + tracker, + ci, + ], + ): + handle = await env.client.start_workflow( + DeliveryWorkflow.run, + args=[spec, recovery], + id=admitted["workflow_id"], + task_queue="issue-243-gates", + ) + result = await asyncio.wait_for(handle.result(), 60) + if spec["policy"]["checks"][0]["argv"][-1].endswith("test_absent"): + assert result["outcome"] == "blocked" + assert result["checks"]["local"]["state"] == "failed" + assert result["checks"]["local"]["results"][0]["exit_code"] != 0 + assert any("local_checks" in finding for finding in result["findings"]) + assert calls == ["publish"] + else: + assert result["outcome"] == "delivered", result + assert calls == ["publish", "review", "verify", "ci"] + assert result["iteration"] == 0 + assert result["roles"][0] == state["roles"][0] + assert ( + result["checks"]["prepublish"]["results"][0]["argv"] + == spec["policy"]["prepublish_checks"][0]["argv"] + ) + assert result["checks"]["local"]["results"][0]["argv"] == spec["policy"]["checks"][0]["argv"] + assert recovery["original_spec"]["policy"]["checks"] == spec["policy"]["checks"] + assert Path(original["readiness"]["receipt"]).read_bytes() == old_receipt + with store._connect() as db: + assert db.execute("SELECT COUNT(*) FROM delivery_attempts").fetchone()[0] == 1 + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + + +@pytest.mark.asyncio +async def test_public_native_retry_reports_missing_descendant_settlement(stopped_readiness): + """A plausible native-shaped checkpoint still cannot bypass missing settlement proof.""" + import hashlib + + from devflow_temporal.delivery_readiness_gate_retry import failure as classify + from devflow_temporal.delivery_readiness_gate_retry import probe_attempt + from devflow_temporal.delivery_resources import write_private + + store, _, state, command, result, closed, _ = stopped_readiness + spec = deepcopy(store.spec("run-1")) + spec["provider"] = "codex" + spec["policy"]["host_sandbox"] = "trusted-local" + spec["policy"]["execution_backend"] = "native-macos" + spec["policy_digest"] = digest(spec["policy"]) + candidate = deepcopy(state["candidate"]) + candidate["policy_digest"] = spec["policy_digest"] + state["candidate"] = candidate + state["roles"][0]["candidate"] = candidate + receipt = result["readiness"] + receipt["binding"]["spec_sha256"] = digest(spec) + receipt["binding"]["candidate"] = candidate + for observation in receipt["observations"].values(): + observation["candidate"] = candidate + receipt["attempt"] = probe_attempt(spec, candidate, 100) + path = ( + Path(spec["state_dir"]) + / "readiness" + / "receipts" + / "plan" + / "0" + / f"{receipt['attempt']}.json" + ) + receipt["receipt"] = str(path) + write_private( + path, + {key: value for key, value in receipt.items() if key not in {"receipt", "receipt_sha256"}}, + ) + receipt["receipt_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + result["candidate_id"] = candidate["id"] + result["failure_class"] = classify(receipt, "prepublish", candidate["id"], "confirmed") + with store._connect() as db: + role_result = { + k: v + for k, v in state["roles"][0].items() + if k not in {"role", "iteration", "candidate"} + } + db.execute( + "UPDATE delivery_attempts SET result_json=? WHERE job_key='original'", + (canonical_json(role_result),), + ) + db.execute( + "UPDATE delivery_runs SET request_json=?,candidate_json=?,checks_json=? " + "WHERE run_id='run-1'", + (canonical_json(spec), canonical_json(candidate), canonical_json(state["checks"])), + ) + closed["result"] = state + async with client_for(store) as (client, headers): + assessment = await client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert assessment.status_code == 200, assessment.text + data = assessment.json() + assert data["eligible"] is False + assert any( + item["code"] == "native_descendant_settlement_unknown" and item["satisfied"] is None + for item in data["blockers"] + ), data + denied = await client.post("/api/runs/run-1/continue-repair", json=command, headers=headers) + assert denied.status_code == 409, denied.text + assert any( + item["code"] == "native_descendant_settlement_unknown" + for item in denied.json()["detail"]["blockers"] + ) + with store._connect() as db: + assert not db.execute("SELECT 1 FROM delivery_gate_admissions").fetchone() + + +@pytest.mark.asyncio +async def test_public_published_readiness_retains_candidate_and_full_selector( + published, monkeypatch +): + store, _, state, closed, _, _ = published + spec = deepcopy(store.spec("run-1")) + spec["readiness_version"] = 1 + spec["resource_cleanup_version"] = 1 + spec["policy"]["readiness"] = { + "version": 1, + "generation": 0, + "browser_engines": [], + "temporal_checks": [], + } + spec["policy_digest"] = digest(spec["policy"]) + with store._connect() as db: + db.execute( + "UPDATE delivery_runs SET request_json=? WHERE run_id=?", + (canonical_json(spec), "run-1"), + ) + broker = DeliveryBroker(store, spec) + resources = RunResources(spec) + staged = broker.checkout.with_name("staged-published-candidate") + broker.checkout.rename(staged) + resources.register(broker.checkout, "checkout") + staged.rename(broker.checkout) + resources.created(broker.checkout) + candidate = broker.candidate() + state["candidate"] = candidate + state["pull_request"]["candidate"] = candidate + state["roles"][-1]["candidate"]["policy_digest"] = spec["policy_digest"] + with store._connect() as db: + role_result = { + k: v + for k, v in state["roles"][-1].items() + if k not in {"role", "iteration", "candidate", "input_candidate_id"} + } + db.execute( + "UPDATE delivery_attempts SET result_json=? WHERE job_key='implementation-1'", + (canonical_json(role_result),), + ) + store.state.release_work(db, spec["work_id"], "external:devflow:run-1") + monkeypatch.setattr( + "devflow_temporal.delivery_preparation.verify_prepared_spec", lambda *_: None + ) + monkeypatch.setattr( + "devflow_temporal.delivery_configured_resources.implementation_prerequisites", lambda *_: [] + ) + monkeypatch.setattr( + "devflow_temporal.delivery_plan_checks.planned_checks", lambda *_a, **_k: [] + ) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, *_a, **_k: { + "state": "failed", + "results": [{"id": "dependency", "passed": False, "cleanup": "confirmed"}], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + readiness = _check_readiness( + broker, {"spec": spec, "candidate": candidate, "iteration": 1}, 101 + ) + assert readiness["failure_class"]["class"] == "environment_prelaunch" + checks = { + "prepublish": { + "state": "passed", + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + "local": readiness, + "readiness": readiness["readiness"], + } + state.update( + checks=checks, cleanup="confirmed", error="execution readiness prevented local checks" + ) + store.project( + "run-1", + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=state["error"], + candidate=candidate, + pull_request=state["pull_request"], + checks=checks, + iteration=1, + protocol_revision=13, + outcome="blocked", + cleanup="confirmed", + error=state["error"], + ) + closed["result"] = state + monkeypatch.setattr(DeliveryStore, "_completed_temporal_result", lambda *_a, **_k: closed) + monkeypatch.setattr("devflow_temporal.delivery_gate_retry._stopped_cleanup", lambda *a, **k: {}) + monkeypatch.setattr("devflow_temporal.delivery_gate_retry.preserve_resources", lambda *_: None) + command = { + "continuation_kind": KIND, + "command_id": "published-readiness-retry", + "expected_revision": 13, + "expected_iteration": 1, + "expected_candidate_id": candidate["id"], + "expected_pr_number": 7, + "expected_pr_head": candidate["head"], + "additional_iterations": 0, + } + with store._connect() as db: + effects = [ + dict(row) for row in db.execute("SELECT * FROM delivery_effects ORDER BY effect_key") + ] + attempts = [ + dict(row) for row in db.execute("SELECT * FROM delivery_attempts ORDER BY job_key") + ] + async with client_for(store) as (client, headers): + narrower = await client.post( + "/api/runs/run-1/repair-admission-preflight", + json={**command, "verification_test_paths": ["test_retry_gate.py"]}, + ) + assert narrower.status_code == 409 and "frozen test selectors" in narrower.text + preflight = await client.post("/api/runs/run-1/repair-admission-preflight", json=command) + assert preflight.status_code == 200, preflight.text + response = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert response.status_code == 200, response.text + assert response.json()["implementation_authority"] is False + execution = store.effective_spec("run-1") + with store._connect() as db: + recovery = json.loads( + db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0] + ) + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_effects ORDER BY effect_key") + ] == effects + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_attempts ORDER BY job_key") + ] == attempts + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert recovery["publication"] == state["pull_request"] + assert recovery["state"]["checks"]["local"] == readiness + assert execution["policy"]["checks"] == spec["policy"]["checks"] + assert readback(store, execution, recovery) == state["pull_request"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stopped_readiness", ["missing-browser"], indirect=True) +async def test_public_retry_retains_missing_browser_runtime_observation(stopped_readiness): + store, _, state, command, original, _, _ = stopped_readiness + observation = original["readiness"]["observations"]["browser"] + assert observation["state"] == "failed" and observation["launched"] is False + assert observation["reason"] == "node unavailable in browser environment" + async with client_for(store) as (client, headers): + response = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert response.status_code == 200, response.text + assert response.json()["implementation_authority"] is False + with store._connect() as db: + recovery = json.loads( + db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0] + ) + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert db.execute("SELECT COUNT(*) FROM delivery_attempts").fetchone()[0] == 1 + assert recovery["state"]["checks"]["prepublish"]["readiness"] == original["readiness"] + assert recovery["candidate"] == state["candidate"] From 31d3f5abf432ae18e3dd40281596def559c73f65 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:19:13 +0100 Subject: [PATCH 06/24] fix(recovery): identify recorded command replays --- docs/temporal-runtime.md | 5 +++ .../devflow_temporal/delivery_gate_retry.py | 13 ++++-- .../devflow_temporal/delivery_lifecycle.py | 43 +++++++++++++++++++ .../delivery_stopped_resume.py | 22 ++++++---- runtime/tests/test_delivery_gate_retry.py | 28 ++++++++++++ .../test_delivery_unpublished_cleanup.py | 30 +++++++++++++ 6 files changed, 129 insertions(+), 12 deletions(-) diff --git a/docs/temporal-runtime.md b/docs/temporal-runtime.md index 7fe0ef7c..6219ab04 100644 --- a/docs/temporal-runtime.md +++ b/docs/temporal-runtime.md @@ -261,6 +261,11 @@ cannot use implementation resume without an accepted plan. Inspect the retained intake and original custody; do not treat a missing plan or unknown cleanup as a completed implementation or a released owner. +When an exact stopped-resume or gate-retry command has already been admitted, +assessment reports `already_admitted` rather than presenting the old receipt as +a fresh grant. Replaying the same command returns its original receipt; reusing +its ID with different inputs reports `command_conflict` and queues no new work. + After that closure, use the public `repair-admission-preflight --id RUN --request FILE` and `continue-repair --id RUN --request FILE` route with `continuation_kind: "stopped_delivery_resume"`, a stable unique `command_id`, the closed protocol `expected_revision`, `expected_iteration`, and current `expected_candidate_id`/`expected_candidate_head`. A passed implementation stopped with exact unknown prepublication cleanup resumes at the sealed iteration's checks with `additional_iterations: 0`. Its exact durable implementation result and unchanged candidate remain mandatory; all publication, independent review, QA, verification, required CI and tracker gates still run. A provider monitor lost after launch remains an unknown verdict. For the narrow original unpublished implementation checkpoint, public cleanup can independently authenticate finished teardown without completing monitoring or inventing a provider session. The closed role, durable supervisor result, exact private request/journal hashes, input candidate, current partial output, provider identity and unknown outcome are bound separately. A later explicit `stopped_delivery_resume` must grant one or two unused original iterations (`additional_iterations: 1` or `2`), starts at the next iteration in a fresh implementation home/session, and receives no invented model finding from the old infrastructure failure. The old result, role and monitoring state remain unchanged. Subsequent normal resumes retain only those exact historical teardown receipts through the immutable admission chain; a different unknown role, changed output/evidence or current live custody still blocks them. Unknown external effects are never replayed. diff --git a/runtime/src/devflow_temporal/delivery_gate_retry.py b/runtime/src/devflow_temporal/delivery_gate_retry.py index 96c1ecf2..e3496678 100644 --- a/runtime/src/devflow_temporal/delivery_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_gate_retry.py @@ -992,8 +992,12 @@ def admit(store, run_id, payload, *, preflight=False): prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() if prior: - if prior['request_digest'] != command_digest: - raise ValueError('command ID already belongs to different inputs') + from .delivery_lifecycle import command_receipt + + assessment = command_receipt(kind, prior, run_id, command_digest) + if preflight: + return {**json.loads(prior['response_json']), 'preflight': True, + 'lifecycle_assessment': assessment} return json.loads(prior['response_json']) seal = snapshot(store, run_id, kind, _observation=preflight) previous = _retained_recovery(store, run_id, seal) @@ -1043,8 +1047,9 @@ def admit(store, run_id, payload, *, preflight=False): prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() if prior: - if prior['request_digest'] != command_digest: - raise ValueError('command ID already belongs to different inputs') + from .delivery_lifecycle import command_receipt + + command_receipt(kind, prior, run_id, command_digest) return json.loads(prior['response_json']) prepared = None if preflight_generation == 1: diff --git a/runtime/src/devflow_temporal/delivery_lifecycle.py b/runtime/src/devflow_temporal/delivery_lifecycle.py index 5eab1e65..2028c4d6 100644 --- a/runtime/src/devflow_temporal/delivery_lifecycle.py +++ b/runtime/src/devflow_temporal/delivery_lifecycle.py @@ -5,6 +5,7 @@ """ from __future__ import annotations +import json import sqlite3 from collections.abc import Mapping from functools import partial @@ -188,6 +189,48 @@ def require_observed_predicates(operation: str, owner: str, proof: Any, raise +def command_receipt(operation: str, prior: Mapping[str, Any], run_id: str, + request_digest: str) -> dict[str, Any]: + """Describe one intact prior admission; conflicting or corrupt receipts refuse.""" + from .contracts import digest + + same = prior['request_digest'] == request_digest + try: + response = json.loads(prior['response_json']) + except (TypeError, ValueError): + response = None + expected_phase = ('repair_continuation_queued' if operation == 'stopped_delivery_resume' + else 'gates_retry_queued') + intact = (same and prior['run_id'] == run_id and isinstance(response, dict) + and response.get('run_id') == run_id and response.get('phase') == expected_phase + and isinstance(response.get('workflow_id'), str) and bool(response['workflow_id'])) + code = ('already_admitted' if intact else 'command_conflict' if not same + else 'command_receipt_unknown') + fact = ('an existing admission for these exact inputs' if intact else + 'a command ID bound to these exact inputs' if not same else + 'an intact saved admission receipt for this run') + assessment = evaluate_transition({ + 'operation': operation, 'owner': 'delivery_commands', + 'owner_proof': True if intact or not same else None, + 'proof_digest': digest({'run_id': prior['run_id'], 'prior': prior['request_digest'], + 'response': prior['response_json'], 'request': request_digest}), + 'comparisons': [], 'predicates': [{ + 'code': code, 'satisfied': False if intact or not same else None, + 'missing_or_unknown_fact': fact, + 'next_action': ('inspect the existing admission' if intact else + 'inspect the saved command and obtain a fresh assessment'), + 'user_input_required': False if intact else None, + }], + }, {}) + assessment['disposition'] = code + if not intact: + exc = RecoveryRefused(assessment) + exc.args = ('command ID already belongs to different inputs' if not same else + 'stored command receipt is unavailable',) + raise exc + return assessment + + def require_same_proof(operation: str, owner: str, observed_digest: str, expected_digest: str) -> dict[str, Any]: """Recheck the owner's full seal under its admission custody.""" diff --git a/runtime/src/devflow_temporal/delivery_stopped_resume.py b/runtime/src/devflow_temporal/delivery_stopped_resume.py index 34ab8aeb..57d008d7 100644 --- a/runtime/src/devflow_temporal/delivery_stopped_resume.py +++ b/runtime/src/devflow_temporal/delivery_stopped_resume.py @@ -1300,12 +1300,16 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): raise ValueError('stopped resume requires a finite exact-checkpoint command') command_digest = digest({'run_id': run_id, **command}) with store._connect() as db: - prior = db.execute('SELECT request_digest,response_json FROM delivery_commands ' + prior = db.execute('SELECT run_id,request_digest,response_json FROM delivery_commands ' 'WHERE command_id=?', (command['command_id'],)).fetchone() if prior: - if prior[0] != command_digest: - raise ValueError('command ID already belongs to different inputs') - return json.loads(prior[1]) + from .delivery_lifecycle import command_receipt + + assessment = command_receipt(KIND, prior, run_id, command_digest) + if preflight: + return {**json.loads(prior['response_json']), 'preflight': True, + 'lifecycle_assessment': assessment} + return json.loads(prior['response_json']) seal = snapshot(store, run_id, _revision=_revision, _observation=preflight) gate_only = None if _revision is not None: @@ -1426,12 +1430,14 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): with _lock(Path(spec['state_dir']) / 'stopped-resumes' / 'controller.lock'): with store._connect() as db: prior = db.execute( - 'SELECT request_digest,response_json FROM delivery_commands WHERE command_id=?', + 'SELECT run_id,request_digest,response_json FROM delivery_commands ' + 'WHERE command_id=?', (command['command_id'],)).fetchone() if prior: - if prior[0] != command_digest: - raise ValueError('command ID already belongs to different inputs') - return json.loads(prior[1]) + from .delivery_lifecycle import command_receipt + + command_receipt(KIND, prior, run_id, command_digest) + return json.loads(prior['response_json']) # Recheck all stopped readbacks after lock acquisition, before any native work. require_same_proof(KIND, 'delivery_stopped_resume', digest(snapshot(store, run_id, _revision=_revision)), digest(seal)) diff --git a/runtime/tests/test_delivery_gate_retry.py b/runtime/tests/test_delivery_gate_retry.py index 83ed4c0e..900a8412 100644 --- a/runtime/tests/test_delivery_gate_retry.py +++ b/runtime/tests/test_delivery_gate_retry.py @@ -322,6 +322,34 @@ async def assess(): replay = await client.post('/api/runs/run-1/continue-repair', json=command, headers=headers) assert replay.json() == accepted.json() + replay_preview = (await assess()).json() + assert not replay_preview['eligible'] + assert replay_preview['disposition'] == 'already_admitted' + assert replay_preview['blockers'][0]['code'] == 'already_admitted' + replay_preflight = await client.post('/api/runs/run-1/repair-admission-preflight', + json=command, headers=headers) + assert replay_preflight.json()['workflow_id'] == accepted.json()['workflow_id'] + assert replay_preflight.json()['additional_iterations'] == 0 + changed_command = {**command, 'expected_revision': command['expected_revision'] + 1} + conflict = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': PREPUBLICATION_KIND, 'request': changed_command}) + rejected = await client.post('/api/runs/run-1/continue-repair', json=changed_command, + headers=headers) + assert conflict.json()['blockers'][0]['code'] == 'command_conflict' + assert rejected.status_code == 409 and rejected.json()['detail'] == conflict.json() + with store._connect() as db: + saved = db.execute('SELECT response_json FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone()[0] + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + ('{}', command['command_id'])) + corrupt = await assess() + refused = await client.post('/api/runs/run-1/continue-repair', + json=command, headers=headers) + assert 'command_receipt_unknown' in {b['code'] for b in corrupt.json()['blockers']} + assert refused.status_code == 409 and refused.json()['detail'] == corrupt.json() + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (saved, command['command_id'])) with store._connect() as db: assert db.execute('SELECT COUNT(*) FROM delivery_gate_admissions').fetchone()[0] == 1 assert state['error'] == 'prepublication repair limit exhausted' diff --git a/runtime/tests/test_delivery_unpublished_cleanup.py b/runtime/tests/test_delivery_unpublished_cleanup.py index 8bc27121..ff181cd0 100644 --- a/runtime/tests/test_delivery_unpublished_cleanup.py +++ b/runtime/tests/test_delivery_unpublished_cleanup.py @@ -206,6 +206,36 @@ async def test_public_unpublished_cleanup_then_existing_finite_continuation(clos assert response.status_code == 200, response.text replay = await client.post('/api/runs/run-1/continue-repair', json=command, headers=headers) assert replay.json() == response.json() + replay_preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + assert replay_preview.json()['disposition'] == 'already_admitted' + assert replay_preview.json()['blockers'][0]['code'] == 'already_admitted' + replay_preflight = await client.post('/api/runs/run-1/repair-admission-preflight', + json=command, headers=headers) + assert replay_preflight.json()['workflow_id'] == response.json()['workflow_id'] + assert replay_preflight.json()['authorized_through_iteration'] == ( + response.json()['authorized_through_iteration']) + conflicting = {**command, 'expected_revision': command['expected_revision'] + 1} + preview_conflict = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': conflicting}) + conflict = await client.post('/api/runs/run-1/continue-repair', json=conflicting, + headers=headers) + assert preview_conflict.json()['blockers'][0]['code'] == 'command_conflict' + assert conflict.status_code == 409 and conflict.json()['detail'] == preview_conflict.json() + with store._connect() as db: + saved = db.execute('SELECT response_json FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone()[0] + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + ('{}', command['command_id'])) + corrupt = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + rejected = await client.post('/api/runs/run-1/continue-repair', json=command, + headers=headers) + assert 'command_receipt_unknown' in {b['code'] for b in corrupt.json()['blockers']} + assert rejected.status_code == 409 and rejected.json()['detail'] == corrupt.json() + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (saved, command['command_id'])) admitted = response.json() # Idempotent replay returns the first admission without another owner or debit. assert admitted['authorized_through_iteration'] == preflight['authorized_through_iteration'] From c7bd401b51a5c2f3a7a07cfc5ad367e1d15c4489 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:22:14 +0100 Subject: [PATCH 07/24] test(recovery): exercise published candidate retry --- .../test_delivery_readiness_gate_retry.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/runtime/tests/test_delivery_readiness_gate_retry.py b/runtime/tests/test_delivery_readiness_gate_retry.py index d56fbbd6..83c12b40 100644 --- a/runtime/tests/test_delivery_readiness_gate_retry.py +++ b/runtime/tests/test_delivery_readiness_gate_retry.py @@ -715,6 +715,117 @@ async def test_public_published_readiness_retains_candidate_and_full_selector( assert execution["policy"]["checks"] == spec["policy"]["checks"] assert readback(store, execution, recovery) == state["pull_request"] + import asyncio + + from temporal_test_server import local_temporal + from temporalio import activity + from temporalio.worker import Worker + + from devflow_temporal.delivery_activities import ( + delivery_checks, + delivery_finalize_resources, + delivery_gates_readback, + delivery_precheck, + delivery_project, + ) + from devflow_temporal.delivery_workflow import DeliveryWorkflow + + store.mark_start("run-1", accepted=True) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, _i, value, **_k: { + "state": "passed", + "results": [], + "candidate_id": value["id"], + "source_unchanged": True, + }, + ) + monkeypatch.setattr( + "devflow_temporal.delivery_readiness.probe_tools", + lambda _broker, requirements, value, **_kw: { + "state": "passed", + "cleanup": "confirmed", + "component": "tools", + "candidate": value, + "requirement_sha256": digest(requirements), + "items": [], + }, + ) + monkeypatch.setattr("devflow_temporal.delivery_readiness_verify.tools", lambda *_a: None) + calls = [] + + @activity.defn(name="delivery_publish") + async def unexpected_publish(_body): + pytest.fail("published retry must never publish the candidate again") + + @activity.defn(name="delivery_role") + async def role(body): + calls.append(body["role"]) + assert body["role"] in {"review", "verify"} + assert body["candidate"] == candidate + return { + "role": body["role"], + "iteration": body["iteration"], + "status": "pass", + "session_id": "independent-" + body["role"], + "candidate": body["candidate"], + "cleanup": "confirmed", + "findings": [], + } + + @activity.defn(name="delivery_tracker_start") + async def start(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_tracker") + async def tracker(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_ci") + async def ci(body): + calls.append("ci") + assert body["pull_request"] == state["pull_request"] + return {"state": "passed"} + + async with local_temporal() as env: + async with Worker( + env.client, + task_queue="issue-243-published", + workflows=[DeliveryWorkflow], + activities=[ + delivery_gates_readback, + delivery_project, + delivery_precheck, + delivery_checks, + delivery_finalize_resources, + unexpected_publish, + role, + start, + tracker, + ci, + ], + ): + handle = await env.client.start_workflow( + DeliveryWorkflow.run, + args=[execution, recovery], + id=response.json()["workflow_id"], + task_queue="issue-243-published", + ) + finished = await asyncio.wait_for(handle.result(), 60) + assert finished["outcome"] == "delivered", finished + assert calls == ["review", "verify", "ci"] + assert finished["candidate"] == candidate + assert finished["checks"]["local"]["results"][0]["argv"] == spec["policy"]["checks"][0]["argv"] + with store._connect() as db: + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_effects ORDER BY effect_key") + ] == effects + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_attempts ORDER BY job_key") + ] == attempts + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + @pytest.mark.asyncio @pytest.mark.parametrize("stopped_readiness", ["missing-browser"], indirect=True) From ed0e162859576d317209ca8903a5eae0fdd14718 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:26:49 +0100 Subject: [PATCH 08/24] fix(recovery): identify stopped candidate drift --- .../delivery_stopped_resume.py | 18 ++++++++++++-- runtime/tests/test_delivery_stopped_resume.py | 24 ++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/runtime/src/devflow_temporal/delivery_stopped_resume.py b/runtime/src/devflow_temporal/delivery_stopped_resume.py index 57d008d7..a0dda293 100644 --- a/runtime/src/devflow_temporal/delivery_stopped_resume.py +++ b/runtime/src/devflow_temporal/delivery_stopped_resume.py @@ -1008,7 +1008,8 @@ def _implementation_receipt(spec, role, attempt): def _candidate(broker, state, attempts, *, missing_rollout=None, input_rejection=None, - recovery=None, pending=None, quarantined=None, historical_quarantines=()): + recovery=None, pending=None, quarantined=None, historical_quarantines=(), + typed_refusal=False, lifecycle='unknown'): candidate = broker.candidate() implementations = [r for r in state.get('roles', []) if r.get('role') == 'implement'] latest = implementations[-1] if implementations else None @@ -1040,6 +1041,18 @@ def _candidate(broker, state, attempts, *, missing_rollout=None, input_rejection and not (missing_rollout or input_rejection or quarantined or retained_unknown)): raise ValueError('blocked implementation has no authenticated launch outcome') elif candidate != state.get('candidate'): + if typed_refusal: + from .delivery_lifecycle import require_observed_predicates + + require_observed_predicates(KIND, 'delivery_stopped_resume', { + 'closed_candidate': state.get('candidate'), 'current_candidate': candidate, + }, [{ + 'code': 'stopped_candidate_changed', 'satisfied': False, + 'missing_or_unknown_fact': 'unchanged candidate at the closed checkpoint', + 'next_action': 'inspect the retained candidate and obtain fresh owner proof', + 'user_input_required': None, + }], lifecycle=lifecycle, owner_proof=None, + legacy_message='stopped candidate changed after its sealed checkpoint') raise ValueError('stopped candidate changed after its sealed checkpoint') if (any((r.get('cleanup') != 'confirmed' or r.get('finish_reason') == 'recovery_unknown') and not (quarantined and r is latest) @@ -1239,7 +1252,8 @@ def _snapshot(store, run_id, *, _revision=None, review_subject=False, _observati candidate, session = _candidate( broker, state, attempts, missing_rollout=(session_custody or {}).get('missing_rollout'), input_rejection=(session_custody or {}).get('input_rejection'), - recovery=previous, pending=pending, quarantined=quarantined, historical_quarantines=history) + recovery=previous, pending=pending, quarantined=quarantined, historical_quarantines=history, + typed_refusal=not review_subject, lifecycle=lifecycle_from_row(row)) publication = state.get('pull_request') if publication: published_identity(broker, candidate, publication) diff --git a/runtime/tests/test_delivery_stopped_resume.py b/runtime/tests/test_delivery_stopped_resume.py index fad409b8..b59c4bcb 100644 --- a/runtime/tests/test_delivery_stopped_resume.py +++ b/runtime/tests/test_delivery_stopped_resume.py @@ -40,7 +40,8 @@ def project(store, state): @pytest.mark.parametrize('drift', ['source', 'zero']) -def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): +@pytest.mark.asyncio +async def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): store, broker, state, command = stopped if drift == 'source': (broker.checkout / 'README.md').write_text('Unsealed post-stop edit\n') @@ -73,6 +74,27 @@ def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): assert not db.execute( "SELECT 1 FROM delivery_commands WHERE command_id='resume-1'").fetchone() assert store.state.claim_for(db, broker.spec['work_id']) is None or drift == 'claim' + import httpx + + from devflow_temporal.delivery_api import create_app + + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), + base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + actual = await client.post('/api/runs/run-1/continue-repair', json=command, + headers={'Origin': host, 'X-Devflow-CSRF': token}) + assert preview.status_code == 200 and not preview.json()['eligible'] + code = 'stopped_candidate_changed' if drift == 'source' else 'pending_repair_unproven' + assert code in {item['code'] for item in preview.json()['blockers']} + assert actual.status_code == 409 and actual.json()['detail'] == preview.json() + with store._connect() as db: + assert not db.execute( + "SELECT 1 FROM delivery_commands WHERE command_id='resume-1'").fetchone() finally: if drift == 'plan': monkey.undo() From 35410b0ba1be9e43b44798953ef62974016e8c6f Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:31:40 +0100 Subject: [PATCH 09/24] fix(recovery): identify stopped custody blockers --- .../delivery_stopped_resume.py | 85 ++++++++++++------- runtime/tests/test_delivery_stopped_resume.py | 30 ++++++- 2 files changed, 83 insertions(+), 32 deletions(-) diff --git a/runtime/src/devflow_temporal/delivery_stopped_resume.py b/runtime/src/devflow_temporal/delivery_stopped_resume.py index a0dda293..5c38e91f 100644 --- a/runtime/src/devflow_temporal/delivery_stopped_resume.py +++ b/runtime/src/devflow_temporal/delivery_stopped_resume.py @@ -1011,17 +1011,35 @@ def _candidate(broker, state, attempts, *, missing_rollout=None, input_rejection recovery=None, pending=None, quarantined=None, historical_quarantines=(), typed_refusal=False, lifecycle='unknown'): candidate = broker.candidate() + + def refuse(code, fact, action, message, *, satisfied=None): + if typed_refusal: + from .delivery_lifecycle import require_observed_predicates + + require_observed_predicates(KIND, 'delivery_stopped_resume', { + 'closed_candidate': state.get('candidate'), 'current_candidate': candidate, + }, [{ + 'code': code, 'satisfied': satisfied, 'missing_or_unknown_fact': fact, + 'next_action': action, 'user_input_required': None, + }], lifecycle=lifecycle, owner_proof=None, legacy_message=message) + raise ValueError(message) + implementations = [r for r in state.get('roles', []) if r.get('role') == 'implement'] latest = implementations[-1] if implementations else None retained_unknown = (latest is not None and pending and pending.get('kind') == 'unstarted_preflight' and digest(latest) in {p['closed_role_sha256'] for p in historical_quarantines}) if latest and latest.get('status') != 'pass': - if ((latest.get('cleanup') != 'confirmed' and not (quarantined or retained_unknown)) - or (latest.get('candidate') != candidate - and not _historical_candidate( - broker, state, candidate, latest, recovery, pending))): - raise ValueError('blocked implementation source or session is unconfirmed') + if latest.get('cleanup') != 'confirmed' and not (quarantined or retained_unknown): + refuse('stopped_implementation_cleanup_unknown', + 'confirmed blocked implementation cleanup or its exact quarantine', + 'reconcile the original implementation custody', + 'blocked implementation source or session is unconfirmed') + if (latest.get('candidate') != candidate + and not _historical_candidate(broker, state, candidate, latest, recovery, pending)): + refuse('stopped_candidate_changed', 'unchanged or authenticated historical candidate', + 'inspect the retained candidate and obtain fresh owner proof', + 'blocked implementation source or session is unconfirmed') matches = [a for a in attempts if a['role'] == 'implement' and a['iteration'] == latest['iteration'] and a['session_id'] == latest['session_id'] @@ -1035,47 +1053,54 @@ def _candidate(broker, state, attempts, *, missing_rollout=None, input_rejection # A closed Temporal activity authenticates controller-only preparation failures. # Every provider or supervisor prelaunch receipt must also match its durable attempt. if len(matches) != 1 and not no_provider and not (quarantined or retained_unknown): - raise ValueError('blocked implementation lacks its exact supervisor receipt') + refuse('stopped_implementation_receipt_unproven', + 'one exact supervisor receipt for the blocked implementation', + 'inspect the original implementation attempt and receipt', + 'blocked implementation lacks its exact supervisor receipt') if latest.get('session_id') is None and not no_provider and ( latest.get('finish_reason') != 'prelaunch' and not (missing_rollout or input_rejection or quarantined or retained_unknown)): - raise ValueError('blocked implementation has no authenticated launch outcome') + refuse('stopped_launch_outcome_unproven', + 'authenticated prelaunch or provider outcome for the blocked implementation', + 'inspect the original launch and provider custody', + 'blocked implementation has no authenticated launch outcome') elif candidate != state.get('candidate'): - if typed_refusal: - from .delivery_lifecycle import require_observed_predicates - - require_observed_predicates(KIND, 'delivery_stopped_resume', { - 'closed_candidate': state.get('candidate'), 'current_candidate': candidate, - }, [{ - 'code': 'stopped_candidate_changed', 'satisfied': False, - 'missing_or_unknown_fact': 'unchanged candidate at the closed checkpoint', - 'next_action': 'inspect the retained candidate and obtain fresh owner proof', - 'user_input_required': None, - }], lifecycle=lifecycle, owner_proof=None, - legacy_message='stopped candidate changed after its sealed checkpoint') - raise ValueError('stopped candidate changed after its sealed checkpoint') - if (any((r.get('cleanup') != 'confirmed' or r.get('finish_reason') == 'recovery_unknown') - and not (quarantined and r is latest) - and digest(r) not in {p['closed_role_sha256'] for p in historical_quarantines} - for r in state.get('roles', [])) - or any(a['state'] != 'finished' or a['cleanup'] != 'confirmed' for a in attempts)): - raise ValueError('stopped role ownership is unknown') + refuse('stopped_candidate_changed', 'unchanged candidate at the closed checkpoint', + 'inspect the retained candidate and obtain fresh owner proof', + 'stopped candidate changed after its sealed checkpoint', satisfied=False) + if any((r.get('cleanup') != 'confirmed' or r.get('finish_reason') == 'recovery_unknown') + and not (quarantined and r is latest) + and digest(r) not in {p['closed_role_sha256'] for p in historical_quarantines} + for r in state.get('roles', [])): + refuse('stopped_role_cleanup_unknown', 'confirmed role cleanup or exact quarantine', + 'reconcile the original role custody', 'stopped role ownership is unknown') + if any(a['state'] != 'finished' or a['cleanup'] != 'confirmed' for a in attempts): + refuse('stopped_attempt_custody_unknown', 'finished attempts and confirmed cleanup', + 'reconcile the original attempt custody', 'stopped role ownership is unknown') sessions = {r.get('session_id') for r in implementations if r.get('session_id')} if len(sessions) > 1: - raise ValueError('original implementation session changed') + refuse('stopped_session_conflict', 'one original implementation session', + 'inspect the retained implementation session history', + 'original implementation session changed', satisfied=False) if not sessions and any(a['role'] == 'implement' and a['session_id'] for a in attempts): - raise ValueError('unrecorded implementation session exists') + refuse('stopped_session_unrecorded', 'matching role and attempt session identity', + 'inspect the retained role and attempt session evidence', + 'unrecorded implementation session exists', satisfied=False) changed = broker._changed_paths(broker.spec['base_sha']) from .delivery_source_scope import outside_scope if outside_scope(broker.spec['policy'], changed, checkout=broker.checkout): - raise ValueError('stopped source escaped its frozen scope') + refuse('stopped_scope_changed', 'source paths within the original authorized scope', + 'inspect changed paths and the original scope authority', + 'stopped source escaped its frozen scope', satisfied=False) _git(broker.checkout, 'merge-base', '--is-ancestor', broker.spec['base_sha'], candidate['head']) if (_git(broker.checkout, 'branch', '--show-current') != broker.spec.get('local_branch', broker.spec['branch']) or _git(broker.checkout, 'remote', 'get-url', '--push', 'origin') != broker.spec['origin_url']): - raise ValueError('stopped checkout branch or origin changed') + refuse('stopped_repository_changed', 'original checkout branch and push destination', + 'inspect the original repository and branch authority', + 'stopped checkout branch or origin changed', satisfied=False) return candidate, next(iter(sessions), None) diff --git a/runtime/tests/test_delivery_stopped_resume.py b/runtime/tests/test_delivery_stopped_resume.py index b59c4bcb..9943f1d2 100644 --- a/runtime/tests/test_delivery_stopped_resume.py +++ b/runtime/tests/test_delivery_stopped_resume.py @@ -39,7 +39,10 @@ def project(store, state): outcome='blocked', cleanup='confirmed', error=state['error']) -@pytest.mark.parametrize('drift', ['source', 'zero']) +@pytest.mark.parametrize('drift', [ + 'source', 'zero', 'cleanup', 'attempt', 'claim', 'unpassed_source', 'receipt', + 'session_conflict', 'unrecorded_session', 'push_origin', 'scope', +]) @pytest.mark.asyncio async def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): store, broker, state, command = stopped @@ -47,6 +50,9 @@ async def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): (broker.checkout / 'README.md').write_text('Unsealed post-stop edit\n') elif drift == 'scope': (broker.checkout / 'foreign.md').write_text('Escaped scope\n') + state['candidate'] = broker.candidate() + state['roles'][0]['candidate'] = state['candidate'] + project(store, state) elif drift == 'claim': with store._connect() as db: store.state.claim_work(db, broker.spec['work_id'], 'external:devflow:run-1', @@ -56,6 +62,17 @@ async def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): elif drift == 'attempt': with store._connect() as db: db.execute("UPDATE delivery_attempts SET state='running'") + elif drift == 'unpassed_source': + state['roles'][0]['status'] = 'blocked' + (broker.checkout / 'README.md').write_text('Unsealed blocked implementation\n') + elif drift == 'receipt': + state['roles'][0]['status'] = 'blocked' + elif drift == 'session_conflict': + state['roles'].append({**state['roles'][0], 'session_id': 'different-session'}) + elif drift == 'unrecorded_session': + state['roles'][0]['session_id'] = None + elif drift == 'push_origin': + _git(broker.checkout, 'remote', 'set-url', '--push', 'origin', '/tmp/foreign-origin.git') elif drift == 'remote': _git(broker.checkout, 'push', 'origin', 'HEAD:refs/heads/' + broker.spec['branch']) elif drift == 'origin': @@ -89,7 +106,16 @@ async def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): actual = await client.post('/api/runs/run-1/continue-repair', json=command, headers={'Origin': host, 'X-Devflow-CSRF': token}) assert preview.status_code == 200 and not preview.json()['eligible'] - code = 'stopped_candidate_changed' if drift == 'source' else 'pending_repair_unproven' + code = { + 'source': 'stopped_candidate_changed', 'zero': 'pending_repair_unproven', + 'cleanup': 'stopped_role_cleanup_unknown', + 'attempt': 'stopped_attempt_custody_unknown', + 'claim': 'stopped_owner_conflict', 'unpassed_source': 'stopped_candidate_changed', + 'receipt': 'stopped_implementation_receipt_unproven', + 'session_conflict': 'stopped_session_conflict', + 'unrecorded_session': 'stopped_session_unrecorded', + 'push_origin': 'stopped_repository_changed', 'scope': 'stopped_scope_changed', + }[drift] assert code in {item['code'] for item in preview.json()['blockers']} assert actual.status_code == 409 and actual.json()['detail'] == preview.json() with store._connect() as db: From c31d6d8fc7e724fab4aab9d168ee292fc93779e6 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:32:56 +0100 Subject: [PATCH 10/24] fix(recovery): authenticate early unreviewed revisions --- docs/trusted-local-delivery.md | 9 + .../delivery_feature_revisions.py | 35 ++++ .../delivery_prelaunch_cleanup.py | 60 ++++--- .../tests/test_delivery_feature_revisions.py | 158 +++++++++++++++--- 4 files changed, 222 insertions(+), 40 deletions(-) diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index 6616d626..0bf6afc4 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -199,6 +199,15 @@ readiness path also requires an already authenticated successor assignment. The current coordinator examines the stopped worker before reserving that assignment, so an unreserved continuation remains blocked rather than bypassing ownership. +For a feature-plan correction stopped before its review launch envelope is +written, the unreviewed-predecessor check can reconstruct the review identity +from the sealed revision, proposal, candidate and owner. It also requires the +supervisor's completed prelaunch result, registered workspace, an empty attempt +directory and the original unheld owner lease. A missing request alone does not +prove the exception, and a real adverse review remains a rejection. Generic +completed-resource observation still requires its existing request evidence for +this early shape. + An older run stopped by that browser-output registration defect can use one `published_check_prelaunch_retry` with zero additional implementation turns. Admission requires its exact closed UNKNOWN checkpoint, passed local checks and diff --git a/runtime/src/devflow_temporal/delivery_feature_revisions.py b/runtime/src/devflow_temporal/delivery_feature_revisions.py index ebbc87e6..c2de5011 100644 --- a/runtime/src/devflow_temporal/delivery_feature_revisions.py +++ b/runtime/src/devflow_temporal/delivery_feature_revisions.py @@ -2261,6 +2261,41 @@ def _unreviewed_rejection(store, spec, context, rejected, values): str(Path(historical["state_dir"]) / "gates" / "0" / "review")), ) request_path = folder / "request.json" + if role == "review" and not os.path.lexists(request_path): + # The revision and proposal identity were reconstructed from + # immutable coordinator checkpoints above, not from a caller. + # A supervisor may refuse preparation before it persists the + # later native launch envelope, so prove the typed terminal + # attempt under its original owner lease instead. + if not attempt: + return None + from .delivery_prelaunch_cleanup import completed_prelaunch + from .delivery_resources import RunResources + + intent = {key: expected[key] for key in ( + "spec", "role", "iteration", "candidate", "resume_session", "revision_context", + "workspace", + )} + manifest = read_private(RunResources(historical).manifest) + with completed_prelaunch(historical, dict(attempt), + {Path(historical["state_dir"]): historical}, manifest, + early_revision_intent=intent) as observed: + with store._connect() as db: + current_attempt = db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", + (owner["run_id"], key), + ).fetchone() + if (not current_attempt or dict(current_attempt) != dict(attempt) + or datetime.fromisoformat(attempt["finished_at"]) > closed_at): + return None + result = json.loads(attempt["result_json"]) + proof = { + "revision_id": revision_id, "review_job_key": key, + "review_intent_digest": observed["revision_intent_sha256"], + "review_result_digest": digest(result), "predecessor_closure": closure, + "prelaunch_evidence": observed, + } + continue if request_path.resolve(strict=True) != request_path: return None request = read_private(request_path) diff --git a/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py b/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py index 8d92dbe9..08ff900d 100644 --- a/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py +++ b/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py @@ -24,12 +24,15 @@ def _stamp(info): @contextmanager -def completed_prelaunch(spec, attempt, executions, manifest, *, completed_intakes=()): +def completed_prelaunch(spec, attempt, executions, manifest, *, completed_intakes=(), + early_revision_intent=None): """Authenticate the existing supervisor's typed pre-monitor failure, never settle it. The durable result is server-owned producer evidence. Neither a missing journal nor a null session suffices. Existing owner/lifecycle leases exclude an unfinished launch; this observer never creates a missing proof or lock. + An early revision identity must be reconstructed by the owning caller from + the sealed predecessor and proposal before it is passed here. """ from .supervisor import DeliverySupervisor @@ -103,20 +106,36 @@ def lease(path, parent): else: locks[name] = lease(folder / name, folder_fd) - fd = os.open('request.json', os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, - dir_fd=folder_fd) - descriptors.append(fd) - info = os.fstat(fd) - if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() - or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 - or info.st_size > _LIMIT): - raise ValueError('completed prelaunch request is not bounded private evidence') - pinned.append((folder / 'request.json', fd, _stamp(info))) - with os.fdopen(os.dup(fd), 'rb') as stream: - raw = stream.read(_LIMIT + 1) - if len(raw) != info.st_size: - raise ValueError('completed prelaunch request changed during observation') - request = json.loads(raw) + early = False + try: + fd = os.open('request.json', os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, + dir_fd=folder_fd) + except FileNotFoundError: + # Only the revision owner may supply a reconstructed identity after + # authenticating its sealed predecessor, proposal and owner records. + # The exact typed supervisor result and the held owner lease above + # establish no launch; an absent request is never sufficient. + if early_revision_intent is None or os.listdir(folder_fd): + raise + early = True + request = early_revision_intent + raw = None + if (not isinstance(request, dict) or request.get('role') != 'review' + or not isinstance(request.get('revision_context'), dict)): + raise ValueError('early prelaunch requires a sealed review identity') from None + else: + descriptors.append(fd) + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() + or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 + or info.st_size > _LIMIT): + raise ValueError('completed prelaunch request is not bounded private evidence') + pinned.append((folder / 'request.json', fd, _stamp(info))) + with os.fdopen(os.dup(fd), 'rb') as stream: + raw = stream.read(_LIMIT + 1) + if len(raw) != info.st_size: + raise ValueError('completed prelaunch request changed during observation') + request = json.loads(raw) if (not isinstance(request, dict) or not isinstance(request.get('spec'), dict) or not isinstance(request.get('candidate'), dict) or not isinstance(request['candidate'].get('id'), str) @@ -136,13 +155,13 @@ def lease(path, parent): if (not admitted or execution.get('provider') != 'codex' or execution.get('policy', {}).get('execution_backend') != 'native-macos' or execution.get('policy_digest') != digest(execution['policy']) - or request.get('native_authorized') is not True + or (not early and request.get('native_authorized') is not True) or DeliverySupervisor._job_key(request) != key or request.get('role') != attempt['role'] or request.get('iteration') != attempt['iteration'] or request['candidate']['id'] != attempt['candidate_id'] - or request.get('result_path') != attempt['result_path'] - or request.get('start_path') != str(folder / 'start.json') + or (not early and (request.get('result_path') != attempt['result_path'] + or request.get('start_path') != str(folder / 'start.json'))) or role not in execution['policy']['roles'] or type(iteration) is not int or iteration < 0 or workspace not in expected_workspaces @@ -154,6 +173,8 @@ def lease(path, parent): raise ValueError('completed prelaunch lost its original admitted request') def unchanged(): + if early and os.listdir(folder_fd): + raise ValueError('early prelaunch attempt is no longer empty') for path, descriptor, stamp in pinned: current, named = os.fstat(descriptor), path.lstat() if any(getattr(current, k) != v or getattr(named, k) != v @@ -172,7 +193,8 @@ def unchanged(): unchanged() yield {'kind': 'completed_native_prelaunch', 'version': 1, 'attempt_sha256': digest(attempt), 'result_sha256': digest(result), - 'request_sha256': hashlib.sha256(raw).hexdigest(), 'leases': locks} + **({'revision_intent_sha256': digest(request)} if early else + {'request_sha256': hashlib.sha256(raw).hexdigest()}), 'leases': locks} unchanged() finally: for fd in reversed(descriptors): diff --git a/runtime/tests/test_delivery_feature_revisions.py b/runtime/tests/test_delivery_feature_revisions.py index 04ba5eaf..610e5791 100644 --- a/runtime/tests/test_delivery_feature_revisions.py +++ b/runtime/tests/test_delivery_feature_revisions.py @@ -2,8 +2,10 @@ from __future__ import annotations +import asyncio import hashlib import json +import os import sys from copy import deepcopy from pathlib import Path @@ -157,7 +159,7 @@ def _revision_diagnostic_for(spec, diagnostic, candidate): def _native_rejected_fixture(native_configuration, monkeypatch, *, prior_cycles=4, - retain_worker=False): + retain_worker=False, early_prelaunch=False): config, request = native_configuration config.raw["max_repairs"] = 10 config.path.write_text(json.dumps(config.raw)) @@ -313,27 +315,49 @@ def observed_build(_store, run_id): **revisions.authenticate_revision_role(store, review), "proposed_plan": value, } - key, _ = supervisor._claim(review) - folder = Path(spec["state_dir"]) / "attempts" / key - review.update( - findings=[], - native_authorized=True, - role_evidence_key=key, - result_path=str(folder / "result.json"), - start_path=str(folder / "start.json"), - workspace=str(Path(spec["state_dir"]) / "gates" / "0" / "review"), - ) - review["review_diff"] = revision_diff(broker, review) - review["review_diff"].pop("candidate_id", None) - write_private(folder / "request.json", review) - with store._connect() as db: - expected = dict( - db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", (key,)).fetchone() + if early_prelaunch: + workspace = broker.gate_checkout("review", 0, candidate) + review["workspace"] = str(workspace) + review["review_diff"] = revision_diff(broker, review) + key = supervisor._job_key(review) + folder = Path(spec["state_dir"]) / "attempts" / key + + def refuse_preparation(*_args, **_kwargs): + raise ValueError("controlled verifier refusal before request persistence") + + def no_process(*_args, **_kwargs): + raise AssertionError("provider was unexpectedly allocated") + + with monkeypatch.context() as refusal: + refusal.setattr("devflow_temporal.delivery_preparation.verify_prepared_spec", + refuse_preparation) + refusal.setattr("devflow_temporal.delivery_native_process.NativeProcess", no_process) + failure = asyncio.run(supervisor.run(review)) + assert failure["finish_reason"] == "prelaunch" and failure["cleanup"] == "confirmed" + assert list(folder.iterdir()) == [] + else: + key, _ = supervisor._claim(review) + folder = Path(spec["state_dir"]) / "attempts" / key + review.update( + findings=[], + native_authorized=True, + role_evidence_key=key, + result_path=str(folder / "result.json"), + start_path=str(folder / "start.json"), + workspace=str(Path(spec["state_dir"]) / "gates" / "0" / "review"), ) - failure = supervisor._native_failure( - review, key, expected, "product patch shape rejected plan comparison", launch_absent=True - ) - assert failure["finish_reason"] == "prelaunch" and failure["cleanup"] == "confirmed" + review["review_diff"] = revision_diff(broker, review) + review["review_diff"].pop("candidate_id", None) + write_private(folder / "request.json", review) + with store._connect() as db: + expected = dict( + db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", (key,)).fetchone() + ) + failure = supervisor._native_failure( + review, key, expected, "product patch shape rejected plan comparison", + launch_absent=True, + ) + assert failure["finish_reason"] == "prelaunch" and failure["cleanup"] == "confirmed" rejection = revisions.reject_revision(store, spec, context["revision_id"], failure["summary"]) row = _stop_revision_fixture(store, spec, shared) return store, spec, shared, value, diagnostic, candidate, context, proposal, rejection, row @@ -578,3 +602,95 @@ def non_gate_drift(*args, **kwargs): revised_worker_spec) response = revisions.revise_feature_plan(store, original["run_id"], command) assert response["repair_budget"]["used"] == 6 + + +@pytest.mark.parametrize("early_prelaunch", [False, True]) +def test_public_revision_retains_unreviewed_prelaunch(native_configuration, monkeypatch, + early_prelaunch): + """A no-launch refusal is not a substantive rejection of a later correction.""" + import httpx + + from devflow_temporal.delivery_api import create_app + + native_configuration[0].raw["execution_mode"] = "trusted-local" + fixture = _native_rejected_fixture(native_configuration, monkeypatch, + early_prelaunch=early_prelaunch) + store, original, shared, plan, diagnostic, _, previous, _, _, row = fixture + command = {"command_id": "prelaunch-revision", "expected_revision": row["revision"], + "reason": "Retry the correction after the environment refusal."} + + async def admit(): + app = create_app(store.config.path) + app.state.delivery.store = store + origin = store.config.dashboard_url.rstrip("/") + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), + base_url=origin) as client: + csrf = (await client.get("/api/session")).json()["csrf_token"] + headers = {"Origin": origin, "X-Devflow-CSRF": csrf} + url = f"/api/runs/{original['run_id']}/revise-feature-plan" + first = await client.post(url, json=command, headers=headers) + replay = await client.post(url, json=command, headers=headers) + assert first.status_code == 200, first.text + assert replay.status_code == 200 and replay.json() == first.json() + return first.json() + + admitted = asyncio.run(admit()) + successor = prepare_authority(store, store.effective_spec(admitted["run_id"])) + broker = DeliveryBroker(store, successor) + broker.prepare() + context = revisions.begin_revision(store, successor) + diagnostic = _revision_diagnostic_for(successor, diagnostic, broker.candidate()) + budget = shared.budget(previous["owner"]["issue_id"]) + assert budget["used"] == 6 + with store._connect() as db: + attempt = dict(db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND role='review'", + (original["run_id"],), + ).fetchone()) + folder = Path(attempt["result_path"]).parent + if early_prelaunch: + assert list(folder.iterdir()) == [] + + def refused(): + with pytest.raises(ValueError, + match="nonprogressing repeated proposal is already rejected"): + revisions.record_proposal(store, successor, context["revision_id"], plan, + diagnostic=diagnostic) + assert shared.budget(previous["owner"]["issue_id"]) == budget + + lease_path = Path(original["state_dir"]) / "attempt-owners" / attempt["job_key"] + assert lease_path.is_file() + lease = DeliverySupervisor._owner_lock(original, attempt["job_key"]) + assert lease is not None + try: + refused() + finally: + os.close(lease) + marker = folder / "launch.json" + marker.write_text("{}") + refused() + marker.unlink() + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET candidate_id=? WHERE job_key=?", + ("forged", attempt["job_key"])) + refused() + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET candidate_id=? WHERE job_key=?", + (attempt["candidate_id"], attempt["job_key"])) + adverse = {**json.loads(attempt["result_json"]), "status": "findings", + "finish_reason": "done"} + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET result_json=? WHERE job_key=?", + (canonical_json(adverse), attempt["job_key"])) + refused() + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET result_json=? WHERE job_key=?", + (attempt["result_json"], attempt["job_key"])) + assert list(folder.iterdir()) == [] + proposal = revisions.record_proposal(store, successor, context["revision_id"], plan, + diagnostic=diagnostic) + assert proposal["unreviewed_predecessors"][0]["revision_id"] == previous["revision_id"] + assert shared.budget(previous["owner"]["issue_id"]) == budget + with store._connect() as db: + assert dict(db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", + (attempt["job_key"],)).fetchone()) == attempt From 49c1375fc70e018e5ade3362fe82d9549e84bac8 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:38:26 +0100 Subject: [PATCH 11/24] fix(recovery): classify early revision prelaunch evidence --- docs/trusted-local-delivery.md | 7 +- .../delivery_feature_revisions.py | 66 +++++++++++-------- .../tests/test_delivery_feature_revisions.py | 10 ++- 3 files changed, 53 insertions(+), 30 deletions(-) diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index 0bf6afc4..348e47c7 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -204,9 +204,10 @@ written, the unreviewed-predecessor check can reconstruct the review identity from the sealed revision, proposal, candidate and owner. It also requires the supervisor's completed prelaunch result, registered workspace, an empty attempt directory and the original unheld owner lease. A missing request alone does not -prove the exception, and a real adverse review remains a rejection. Generic -completed-resource observation still requires its existing request evidence for -this early shape. +prove the exception, and a real adverse review remains a rejection. The strict +reconsideration reader classifies the same evidence as prelaunch, never as an +independent review. Generic completed-resource observation still requires its +existing request evidence for this early shape. An older run stopped by that browser-output registration defect can use one `published_check_prelaunch_retry` with zero additional implementation turns. diff --git a/runtime/src/devflow_temporal/delivery_feature_revisions.py b/runtime/src/devflow_temporal/delivery_feature_revisions.py index c2de5011..f2782db1 100644 --- a/runtime/src/devflow_temporal/delivery_feature_revisions.py +++ b/runtime/src/devflow_temporal/delivery_feature_revisions.py @@ -1652,6 +1652,31 @@ def _private_canonical(path, expected): return hashlib.sha256(encoded).hexdigest() +def _early_revision_prelaunch(store, spec, attempt, expected, *, closed_at=None): + """Observe a sealed revision review that stopped before persisting its envelope.""" + from .delivery_prelaunch_cleanup import completed_prelaunch + from .delivery_resources import RunResources, read_private + + intent = {key: expected[key] for key in ( + "spec", "role", "iteration", "candidate", "resume_session", "revision_context", + "workspace", + )} + manifest = read_private(RunResources(spec).manifest) + with completed_prelaunch(spec, dict(attempt), {Path(spec["state_dir"]): spec}, manifest, + early_revision_intent=intent) as observed: + with store._connect() as db: + current = db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", + (spec["run_id"], attempt["job_key"]), + ).fetchone() + finished = datetime.fromisoformat(attempt["finished_at"]) + if (not current or dict(current) != dict(attempt) or finished.tzinfo is None + or (closed_at is not None and finished > closed_at)): + raise ValueError("early revision attempt changed or did not precede closure") + result = json.loads(attempt["result_json"]) + return result, observed + + def _saved_revision_role(store, spec, context, proposal, candidate, role, *, closed_at=None, shared=None): """Authenticate the exact supervised launcher and actual completed provider.""" @@ -1692,6 +1717,13 @@ def _saved_revision_role(store, spec, context, proposal, candidate, role, *, clo if steering and json.loads(steering[0]): expected["steering"] = json.loads(steering[0]) request_path = folder / "request.json" + if role == "review" and not os.path.lexists(request_path): + result, observed = _early_revision_prelaunch(store, spec, saved, expected, + closed_at=closed_at) + return result, {"run_id": spec["run_id"], "job_key": key, + "intent_sha256": observed["revision_intent_sha256"], + "result_sha256": digest(result), "session_id": None, + "classification": "prelaunch", "prelaunch_evidence": observed} if request_path.resolve(strict=True) != request_path: raise ValueError("historical native request left its exact owned path") request = read_private(request_path) @@ -2269,32 +2301,14 @@ def _unreviewed_rejection(store, spec, context, rejected, values): # attempt under its original owner lease instead. if not attempt: return None - from .delivery_prelaunch_cleanup import completed_prelaunch - from .delivery_resources import RunResources - - intent = {key: expected[key] for key in ( - "spec", "role", "iteration", "candidate", "resume_session", "revision_context", - "workspace", - )} - manifest = read_private(RunResources(historical).manifest) - with completed_prelaunch(historical, dict(attempt), - {Path(historical["state_dir"]): historical}, manifest, - early_revision_intent=intent) as observed: - with store._connect() as db: - current_attempt = db.execute( - "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", - (owner["run_id"], key), - ).fetchone() - if (not current_attempt or dict(current_attempt) != dict(attempt) - or datetime.fromisoformat(attempt["finished_at"]) > closed_at): - return None - result = json.loads(attempt["result_json"]) - proof = { - "revision_id": revision_id, "review_job_key": key, - "review_intent_digest": observed["revision_intent_sha256"], - "review_result_digest": digest(result), "predecessor_closure": closure, - "prelaunch_evidence": observed, - } + result, observed = _early_revision_prelaunch( + store, historical, attempt, expected, closed_at=closed_at) + proof = { + "revision_id": revision_id, "review_job_key": key, + "review_intent_digest": observed["revision_intent_sha256"], + "review_result_digest": digest(result), "predecessor_closure": closure, + "prelaunch_evidence": observed, + } continue if request_path.resolve(strict=True) != request_path: return None diff --git a/runtime/tests/test_delivery_feature_revisions.py b/runtime/tests/test_delivery_feature_revisions.py index 610e5791..1a5fb5af 100644 --- a/runtime/tests/test_delivery_feature_revisions.py +++ b/runtime/tests/test_delivery_feature_revisions.py @@ -615,7 +615,7 @@ def test_public_revision_retains_unreviewed_prelaunch(native_configuration, monk native_configuration[0].raw["execution_mode"] = "trusted-local" fixture = _native_rejected_fixture(native_configuration, monkeypatch, early_prelaunch=early_prelaunch) - store, original, shared, plan, diagnostic, _, previous, _, _, row = fixture + store, original, shared, plan, diagnostic, candidate, previous, prior_proposal, _, row = fixture command = {"command_id": "prelaunch-revision", "expected_revision": row["revision"], "reason": "Retry the correction after the environment refusal."} @@ -649,6 +649,14 @@ async def admit(): ).fetchone()) folder = Path(attempt["result_path"]).parent if early_prelaunch: + from datetime import UTC, datetime + + review_result, review_proof = revisions._saved_revision_role( + store, original, previous, prior_proposal, candidate, "review", + closed_at=datetime(2099, 1, 1, tzinfo=UTC), + ) + assert review_proof["classification"] == "prelaunch" + assert review_result["status"] == "blocked" assert list(folder.iterdir()) == [] def refused(): From beb6994c3eaa26abc510d8c62165aa696a861002 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:39:35 +0100 Subject: [PATCH 12/24] fix(recovery): report publication custody and receipts --- docs/temporal-runtime.md | 11 ++++--- .../devflow_temporal/delivery_lifecycle.py | 6 ++-- .../delivery_pending_publication.py | 30 +++++++++++++------ .../test_delivery_pending_publication.py | 27 +++++++++++++++++ .../test_delivery_publication_self_heal.py | 26 ++++++++++++++++ 5 files changed, 85 insertions(+), 15 deletions(-) diff --git a/docs/temporal-runtime.md b/docs/temporal-runtime.md index 6219ab04..9ccb99ca 100644 --- a/docs/temporal-runtime.md +++ b/docs/temporal-runtime.md @@ -261,10 +261,13 @@ cannot use implementation resume without an accepted plan. Inspect the retained intake and original custody; do not treat a missing plan or unknown cleanup as a completed implementation or a released owner. -When an exact stopped-resume or gate-retry command has already been admitted, -assessment reports `already_admitted` rather than presenting the old receipt as -a fresh grant. Replaying the same command returns its original receipt; reusing -its ID with different inputs reports `command_conflict` and queues no new work. +When an exact stopped-resume, gate-retry, or pending-publication command has +already been admitted, assessment reports `already_admitted` rather than presenting +the old receipt as a fresh grant. Replaying the same command returns its original +receipt; reusing its ID with different inputs reports `command_conflict` and queues +no new work. An unverified stored receipt remains unknown. Pending publication also +reports unknown native resource custody when the original process, root, or lease +cannot be proven settled; the existing observer still refuses admission. After that closure, use the public `repair-admission-preflight --id RUN --request FILE` and `continue-repair --id RUN --request FILE` route with `continuation_kind: "stopped_delivery_resume"`, a stable unique `command_id`, the closed protocol `expected_revision`, `expected_iteration`, and current `expected_candidate_id`/`expected_candidate_head`. A passed implementation stopped with exact unknown prepublication cleanup resumes at the sealed iteration's checks with `additional_iterations: 0`. Its exact durable implementation result and unchanged candidate remain mandatory; all publication, independent review, QA, verification, required CI and tracker gates still run. diff --git a/runtime/src/devflow_temporal/delivery_lifecycle.py b/runtime/src/devflow_temporal/delivery_lifecycle.py index 2028c4d6..901aac41 100644 --- a/runtime/src/devflow_temporal/delivery_lifecycle.py +++ b/runtime/src/devflow_temporal/delivery_lifecycle.py @@ -199,8 +199,10 @@ def command_receipt(operation: str, prior: Mapping[str, Any], run_id: str, response = json.loads(prior['response_json']) except (TypeError, ValueError): response = None - expected_phase = ('repair_continuation_queued' if operation == 'stopped_delivery_resume' - else 'gates_retry_queued') + expected_phase = { + 'stopped_delivery_resume': 'repair_continuation_queued', + 'recover_publication': 'publication_recovery_queued', + }.get(operation, 'gates_retry_queued') intact = (same and prior['run_id'] == run_id and isinstance(response, dict) and response.get('run_id') == run_id and response.get('phase') == expected_phase and isinstance(response.get('workflow_id'), str) and bool(response['workflow_id'])) diff --git a/runtime/src/devflow_temporal/delivery_pending_publication.py b/runtime/src/devflow_temporal/delivery_pending_publication.py index 1c44625e..ce277d1c 100644 --- a/runtime/src/devflow_temporal/delivery_pending_publication.py +++ b/runtime/src/devflow_temporal/delivery_pending_publication.py @@ -29,7 +29,7 @@ def _chain(recovery): recovery = recovery.get('original_recovery') -def _refuse(code, fact, action, proof, *, message=None): +def _refuse(code, fact, action, proof, *, message=None, satisfied=False): from .delivery_lifecycle import RecoveryRefused, require_transition try: @@ -37,7 +37,7 @@ def _refuse(code, fact, action, proof, *, message=None): 'operation': 'recover_publication', 'owner': 'pending_publication', 'owner_proof': True, 'normalized_lifecycle': 'stopped', 'proof_digest': digest(proof), 'comparisons': [], 'predicates': [{ - 'code': code, 'satisfied': False, 'missing_or_unknown_fact': fact, + 'code': code, 'satisfied': satisfied, 'missing_or_unknown_fact': fact, 'next_action': action, }], }, {}) @@ -231,8 +231,18 @@ def snapshot(store, run_id, *, observation=False): message='original publication executor or remote outcome remains unknown') from .delivery_resources import observe_completed_resources - settlement = {'publication': publication, - 'native_cleanup': observe_completed_resources(spec, attempts)} + try: + native_cleanup = observe_completed_resources(spec, attempts) + except (ValueError, OSError) as exc: + from .delivery_lifecycle import RecoveryRefused + + if isinstance(exc, RecoveryRefused): + raise + _refuse('publication_native_custody_unknown', + 'settled native actors, roots and resource leases', + 'reconcile the original native resource custody', + {'row': row, 'closed': closed}, message=str(exc), satisfied=None) + settlement = {'publication': publication, 'native_cleanup': native_cleanup} with store._connect() as db: binding = work_binding(store, spec, db) return {'row': _sealed_row(row), 'closed': closed, 'spec': spec, @@ -257,17 +267,20 @@ def admit(store, run_id, payload, *, preflight=False): r'[0-9a-f]{' + str(n) + '}', payload[k]) for k, n in (('expected_candidate_id', 64), ('expected_head', 40)))): raise ValueError('invalid pending publication retry identity') + from .delivery_lifecycle import command_receipt, require_same_proof, require_transition + command_digest = digest({'run_id': run_id, **payload}) with store._connect() as db: prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() if prior: - if prior['request_digest'] != command_digest: - raise ValueError('command ID already belongs to different inputs') + assessment = command_receipt('recover_publication', prior, run_id, command_digest) + if preflight: + return {**json.loads(prior['response_json']), 'preflight': True, + 'lifecycle_assessment': assessment} return json.loads(prior['response_json']) seal = snapshot(store, run_id, observation=preflight) previous = _retained_recovery(store, run_id, seal) - from .delivery_lifecycle import require_same_proof, require_transition assessment = require_transition({ 'operation': 'recover_publication', 'owner': 'pending_publication', 'owner_proof': True, @@ -290,8 +303,7 @@ def admit(store, run_id, payload, *, preflight=False): prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() if prior: - if prior['request_digest'] != command_digest: - raise ValueError('command ID already belongs to different inputs') + command_receipt('recover_publication', prior, run_id, command_digest) return json.loads(prior['response_json']) execution = prepare_runtime(spec, root, command_digest, digest(seal)) require_same_proof('recover_publication', 'pending_publication', diff --git a/runtime/tests/test_delivery_pending_publication.py b/runtime/tests/test_delivery_pending_publication.py index 6b1d3c5c..acae17af 100644 --- a/runtime/tests/test_delivery_pending_publication.py +++ b/runtime/tests/test_delivery_pending_publication.py @@ -120,3 +120,30 @@ async def test_public_pending_assessment_agrees_with_admission(pending): assert refused.json()['blockers'][0]['code'] == 'revision_changed' accepted = await client.post(path + 'recover-publication', json=request, headers=headers) assert accepted.status_code == 200 and not accepted.json()['implementation_authority'] + replay = await client.post(path + 'recover-publication', json=request, headers=headers) + assert replay.status_code == 200 and replay.json() == accepted.json() + replay_preview = await client.post(path + 'assess-transition', json={ + 'kind': 'recover_publication', 'request': request}) + conflict = {**request, 'expected_revision': request['expected_revision'] + 1} + conflict_preview = await client.post(path + 'assess-transition', json={ + 'kind': 'recover_publication', 'request': conflict}) + conflict_admission = await client.post(path + 'recover-publication', json=conflict, + headers=headers) + assert conflict_admission.status_code == 409 + assert conflict_admission.json()['detail'] == conflict_preview.json(), ( + conflict_preview.json(), conflict_admission.json()) + assert conflict_preview.json()['blockers'][0]['code'] == 'command_conflict' + assert replay_preview.json()['disposition'] == 'already_admitted' + assert replay_preview.json()['blockers'][0]['code'] == 'already_admitted' + with store._connect() as db: + assert db.execute('SELECT count(*) FROM delivery_commands WHERE command_id=?', + (request['command_id'],)).fetchone()[0] == 1 + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + ('{}', request['command_id'])) + corrupt_preview = await client.post(path + 'assess-transition', json={ + 'kind': 'recover_publication', 'request': request}) + corrupt_admission = await client.post(path + 'recover-publication', json=request, + headers=headers) + assert corrupt_admission.status_code == 409 + assert corrupt_admission.json()['detail'] == corrupt_preview.json() + assert 'command_receipt_unknown' in {b['code'] for b in corrupt_preview.json()['blockers']} diff --git a/runtime/tests/test_delivery_publication_self_heal.py b/runtime/tests/test_delivery_publication_self_heal.py index 3bdc6bf6..90810fdc 100644 --- a/runtime/tests/test_delivery_publication_self_heal.py +++ b/runtime/tests/test_delivery_publication_self_heal.py @@ -310,6 +310,32 @@ async def terminal_tracker(payload): # PR readback; the publication lease alone is not native cleanup. actor = subprocess.Popen(['/bin/sleep', '30'], cwd=broker.checkout) try: + import httpx + + from devflow_temporal.delivery_api import create_app + + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + transport = httpx.ASGITransport(app=app, client=('127.0.0.1', 10001)) + async with httpx.AsyncClient(transport=transport, base_url=host) as browser: + token = (await browser.get('/api/session')).json()['csrf_token'] + headers = {'Origin': host, 'X-Devflow-CSRF': token} + path = '/api/runs/run-1/' + preflight = await browser.post(path + 'assess-transition', json={ + 'kind': 'recover_publication', 'request': recover}) + refusal = await browser.post(path + 'recover-publication', json=recover, + headers=headers) + assert preflight.status_code == 200 and not preflight.json()['eligible'] + assert any(item['code'] == 'publication_native_custody_unknown' + and item['satisfied'] is None + for item in preflight.json()['blockers']), ( + preflight.json(), refusal.status_code, refusal.json()) + assert refusal.status_code == 409 and refusal.json()['detail'] == preflight.json() + with store._connect() as db: + assert not db.execute( + 'SELECT 1 FROM delivery_commands WHERE command_id=?', + (recover['command_id'],)).fetchone() with pytest.raises(ValueError, match='live root user|unreadable lease'): await asyncio.to_thread(store.recover_publication, 'run-1', recover) finally: From 70ce906148a1faaf16a7404b3fa0127a9d689373 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:42:43 +0100 Subject: [PATCH 13/24] fix(recovery): lock prelaunch resource evidence --- .../delivery_feature_revisions.py | 29 ++++++++++--------- .../tests/test_delivery_feature_revisions.py | 25 +++++++++++++--- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/runtime/src/devflow_temporal/delivery_feature_revisions.py b/runtime/src/devflow_temporal/delivery_feature_revisions.py index f2782db1..fdcd9cc0 100644 --- a/runtime/src/devflow_temporal/delivery_feature_revisions.py +++ b/runtime/src/devflow_temporal/delivery_feature_revisions.py @@ -1655,25 +1655,26 @@ def _private_canonical(path, expected): def _early_revision_prelaunch(store, spec, attempt, expected, *, closed_at=None): """Observe a sealed revision review that stopped before persisting its envelope.""" from .delivery_prelaunch_cleanup import completed_prelaunch - from .delivery_resources import RunResources, read_private + from .delivery_resources import RunResources intent = {key: expected[key] for key in ( "spec", "role", "iteration", "candidate", "resume_session", "revision_context", "workspace", )} - manifest = read_private(RunResources(spec).manifest) - with completed_prelaunch(spec, dict(attempt), {Path(spec["state_dir"]): spec}, manifest, - early_revision_intent=intent) as observed: - with store._connect() as db: - current = db.execute( - "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", - (spec["run_id"], attempt["job_key"]), - ).fetchone() - finished = datetime.fromisoformat(attempt["finished_at"]) - if (not current or dict(current) != dict(attempt) or finished.tzinfo is None - or (closed_at is not None and finished > closed_at)): - raise ValueError("early revision attempt changed or did not precede closure") - result = json.loads(attempt["result_json"]) + registry = RunResources(spec, read_only=True) + with registry.locked(read_only=True) as manifest: + with completed_prelaunch(spec, dict(attempt), {Path(spec["state_dir"]): spec}, manifest, + early_revision_intent=intent) as observed: + with store._connect() as db: + current = db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", + (spec["run_id"], attempt["job_key"]), + ).fetchone() + finished = datetime.fromisoformat(attempt["finished_at"]) + if (not current or dict(current) != dict(attempt) or finished.tzinfo is None + or (closed_at is not None and finished > closed_at)): + raise ValueError("early revision attempt changed or did not precede closure") + result = json.loads(attempt["result_json"]) return result, observed diff --git a/runtime/tests/test_delivery_feature_revisions.py b/runtime/tests/test_delivery_feature_revisions.py index 1a5fb5af..417f52b0 100644 --- a/runtime/tests/test_delivery_feature_revisions.py +++ b/runtime/tests/test_delivery_feature_revisions.py @@ -649,12 +649,29 @@ async def admit(): ).fetchone()) folder = Path(attempt["result_path"]).parent if early_prelaunch: + from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime + from threading import Event - review_result, review_proof = revisions._saved_revision_role( - store, original, previous, prior_proposal, candidate, "review", - closed_at=datetime(2099, 1, 1, tzinfo=UTC), - ) + from devflow_temporal.delivery_resources import RunResources + + started = Event() + + def read_review(): + started.set() + return revisions._saved_revision_role( + store, original, previous, prior_proposal, candidate, "review", + closed_at=datetime(2099, 1, 1, tzinfo=UTC), + ) + + with ThreadPoolExecutor(max_workers=1) as executor: + registry = RunResources(original, read_only=True) + with registry.locked(read_only=True, exclusive=True): + pending = executor.submit(read_review) + assert started.wait(5) + with pytest.raises(TimeoutError): + pending.result(timeout=0.2) + review_result, review_proof = pending.result(timeout=15) assert review_proof["classification"] == "prelaunch" assert review_result["status"] == "blocked" assert list(folder.iterdir()) == [] From 882d162190725aadc9a1ff17cc955a9c7812626d Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:49:55 +0100 Subject: [PATCH 14/24] fix(recovery): retain mixed revision review history --- docs/trusted-local-delivery.md | 6 +- .../delivery_feature_revisions.py | 9 ++- .../tests/test_delivery_feature_revisions.py | 75 +++++++++++++++++++ 3 files changed, 85 insertions(+), 5 deletions(-) diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index 348e47c7..ef1c0e7e 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -206,8 +206,10 @@ supervisor's completed prelaunch result, registered workspace, an empty attempt directory and the original unheld owner lease. A missing request alone does not prove the exception, and a real adverse review remains a rejection. The strict reconsideration reader classifies the same evidence as prelaunch, never as an -independent review. Generic completed-resource observation still requires its -existing request evidence for this early shape. +independent review. When a later genuine review is reconsidered, its history +retains the earlier prelaunch intent digest without inventing a request digest. +Generic completed-resource observation still requires its existing request +evidence for this early shape. An older run stopped by that browser-output registration defect can use one `published_check_prelaunch_retry` with zero additional implementation turns. diff --git a/runtime/src/devflow_temporal/delivery_feature_revisions.py b/runtime/src/devflow_temporal/delivery_feature_revisions.py index fdcd9cc0..d61b54f2 100644 --- a/runtime/src/devflow_temporal/delivery_feature_revisions.py +++ b/runtime/src/devflow_temporal/delivery_feature_revisions.py @@ -2044,10 +2044,13 @@ def _prove_reconsideration(store, spec, selectors, record, closure, values, *, p _text(text, "unresolved prior finding", maximum=16 * 1024) if text not in findings: findings.append(text) + review = proof["review"] + review_identity = ({"review_request_sha256": review["request_sha256"]} + if "request_sha256" in review else + {"review_intent_sha256": review["intent_sha256"]}) predecessors.append({"revision_id": proof["revision_id"], - "classification": proof["classification"], - "review_request_sha256": proof["review"]["request_sha256"], - "review_result_sha256": proof["review"]["result_sha256"]}) + "classification": proof["classification"], **review_identity, + "review_result_sha256": review["result_sha256"]}) product_findings = [text for worker in subject["worker_summaries"] for text in (worker.get("current_review_subject") or {}).get( "unresolved_product_findings", [])] diff --git a/runtime/tests/test_delivery_feature_revisions.py b/runtime/tests/test_delivery_feature_revisions.py index 417f52b0..d9aad92d 100644 --- a/runtime/tests/test_delivery_feature_revisions.py +++ b/runtime/tests/test_delivery_feature_revisions.py @@ -719,3 +719,78 @@ def refused(): with store._connect() as db: assert dict(db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", (attempt["job_key"],)).fetchone()) == attempt + +def test_public_reconsideration_preserves_early_and_genuine_reviews( + native_configuration, monkeypatch, tmp_path, +): + """An unlaunched review stays unreviewed beside a later genuine rejection.""" + import httpx + + from devflow_temporal.delivery_api import create_app + + native_configuration[0].raw['execution_mode'] = 'trusted-local' + fixture = _native_rejected_fixture(native_configuration, monkeypatch, + retain_worker='build', early_prelaunch=True) + store, original, shared, plan, diagnostic, _, prior, _, rejection, row = fixture + _, _, historical, current = context_repository(tmp_path, monkeypatch) + _complete_native_role_fixture(store, original, prior, plan, diagnostic, 'intake', historical) + command = {'command_id': 'mixed-ordinary-revision', 'expected_revision': row['revision'], + 'reason': 'Retry an unreviewed plan correction.'} + + async def post(run_id, command): + app = create_app(store.config.path) + app.state.delivery.store = store + origin = store.config.dashboard_url.rstrip('/') + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=origin) as c: + csrf = (await c.get('/api/session')).json()['csrf_token'] + return await c.post(f'/api/runs/{run_id}/revise-feature-plan', json=command, + headers={'Origin': origin, 'X-Devflow-CSRF': csrf}) + + early_selectors = {'version': 1, 'kind': 'derivation_context_change', + 'rejected_revision_id': prior['revision_id'], + 'proposal_sha256': digest(plan), + 'expected_runtime_revision': current['revision'], + 'expected_runtime_payload_sha256': current['runtime_payload_sha256']} + early_command = {**command, 'command_id': 'reject-unreviewed-reconsideration', + 'reconsideration': early_selectors} + denied = asyncio.run(post(original['run_id'], early_command)) + assert denied.status_code == 409, denied.text + assert 'genuine completed independent review' in denied.text + assert shared.budget(prior['owner']['issue_id'])['used'] == 5 + response = asyncio.run(post(original['run_id'], command)) + assert response.status_code == 200, response.text + successor = prepare_authority(store, store.effective_spec(response.json()['run_id'])) + broker = DeliveryBroker(store, successor) + broker.prepare() + with store._connect() as db: + db.execute('UPDATE delivery_runs SET candidate_json=? WHERE run_id=?', + (canonical_json(broker.candidate()), successor['run_id'])) + context = revisions.begin_revision(store, successor) + diagnostic = _revision_diagnostic_for(successor, diagnostic, broker.candidate()) + _complete_native_role_fixture(store, successor, context, plan, diagnostic, 'intake', historical) + proposal = revisions.record_proposal(store, successor, context['revision_id'], plan, + diagnostic=diagnostic) + assert proposal['unreviewed_predecessors'][0]['revision_id'] == prior['revision_id'] + _complete_native_role_fixture(store, successor, context, plan, diagnostic, 'review', historical) + rejected = revisions.reject_revision(store, successor, context['revision_id'], + 'A genuine independent review found a defect.') + stopped = _stop_revision_fixture(store, successor, shared) + selectors = {'version': 1, 'kind': 'derivation_context_change', + 'rejected_revision_id': context['revision_id'], 'proposal_sha256': digest(plan), + 'expected_runtime_revision': current['revision'], + 'expected_runtime_payload_sha256': current['runtime_payload_sha256']} + command = {'command_id':'mixed-reconsideration','expected_revision':stopped['revision'], + 'reason':'Reconsider after the verified derivation change.', + 'reconsideration':selectors} + response = asyncio.run(post(successor['run_id'], command)) + assert response.status_code == 200, response.text + proofs = response.json()['reconsideration']['prior_rejections'] + earlier = next(x for x in proofs if x['revision_id'] == prior['revision_id']) + later = next(x for x in proofs if x['revision_id'] == context['revision_id']) + assert earlier['classification'] == 'prelaunch' and len(earlier['review_intent_sha256']) == 64 + assert 'review_request_sha256' not in earlier + assert later['classification'] == 'genuine' and len(later['review_request_sha256']) == 64 + assert response.json()['repair_budget']['used'] == 6 + checkpoints = shared.checkpoints(prior['owner']['issue_id']) + assert checkpoints[revisions.PREFIX + 'rejected:' + prior['revision_id']] == rejection + assert checkpoints[revisions.PREFIX + 'rejected:' + context['revision_id']] == rejected From e72e5ab5c72680fb986035d15350751c90e4dbb8 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:50:37 +0100 Subject: [PATCH 15/24] fix(recovery): share historical repair admission checks --- docs/temporal-runtime.md | 5 +- .../devflow_temporal/delivery_lifecycle.py | 1 + .../src/devflow_temporal/delivery_store.py | 192 +++++++++++------- .../test_delivery_store_lifecycle_240.py | 84 +++++++- 4 files changed, 207 insertions(+), 75 deletions(-) diff --git a/docs/temporal-runtime.md b/docs/temporal-runtime.md index 9ccb99ca..9271cd2a 100644 --- a/docs/temporal-runtime.md +++ b/docs/temporal-runtime.md @@ -261,8 +261,9 @@ cannot use implementation resume without an accepted plan. Inspect the retained intake and original custody; do not treat a missing plan or unknown cleanup as a completed implementation or a released owner. -When an exact stopped-resume, gate-retry, or pending-publication command has -already been admitted, assessment reports `already_admitted` rather than presenting +When an exact ordinary repair, stopped-resume, gate-retry, or pending-publication +command has already been admitted, assessment reports `already_admitted` rather +than presenting the old receipt as a fresh grant. Replaying the same command returns its original receipt; reusing its ID with different inputs reports `command_conflict` and queues no new work. An unverified stored receipt remains unknown. Pending publication also diff --git a/runtime/src/devflow_temporal/delivery_lifecycle.py b/runtime/src/devflow_temporal/delivery_lifecycle.py index 901aac41..214038e8 100644 --- a/runtime/src/devflow_temporal/delivery_lifecycle.py +++ b/runtime/src/devflow_temporal/delivery_lifecycle.py @@ -201,6 +201,7 @@ def command_receipt(operation: str, prior: Mapping[str, Any], run_id: str, response = None expected_phase = { 'stopped_delivery_resume': 'repair_continuation_queued', + 'repair_continuation': 'repair_continuation_queued', 'recover_publication': 'publication_recovery_queued', }.get(operation, 'gates_retry_queued') intact = (same and prior['run_id'] == run_id and isinstance(response, dict) diff --git a/runtime/src/devflow_temporal/delivery_store.py b/runtime/src/devflow_temporal/delivery_store.py index e2f11692..e124e7d1 100644 --- a/runtime/src/devflow_temporal/delivery_store.py +++ b/runtime/src/devflow_temporal/delivery_store.py @@ -1951,6 +1951,11 @@ def continue_repair( return json.loads(prior["response_json"]) raise ValueError("repair continuation fields do not match the contract") from .delivery_broker import DeliveryBroker + from .delivery_lifecycle import ( + command_receipt, + require_observed_predicates, + require_transition, + ) from .delivery_repair import ( confirmed_native_cleanup, failed_gate_diagnostics, @@ -1983,12 +1988,14 @@ def continue_repair( command_digest = digest({"run_id": run_id, **supplied}) with self._connect() as db: prior = db.execute( - "SELECT request_digest,response_json FROM delivery_commands WHERE command_id=?", - (command_id,), + "SELECT run_id,request_digest,response_json FROM delivery_commands " + "WHERE command_id=?", (command_id,), ).fetchone() if prior: - if prior["request_digest"] != command_digest: - raise ValueError("command ID already belongs to different inputs") + receipt = command_receipt('repair_continuation', prior, run_id, command_digest) + if preflight: + return {**json.loads(prior["response_json"]), "preflight": True, + "lifecycle_assessment": receipt} return json.loads(prior["response_json"]) row = db.execute("SELECT * FROM delivery_runs WHERE run_id=?", (run_id,)).fetchone() granted = db.execute( @@ -1997,7 +2004,13 @@ def continue_repair( if row is None: raise ValueError("run ID not found") if granted: - raise ValueError("this run already received its one repair grant") + require_observed_predicates('repair_continuation', 'repair_admission', + {'row': dict(row), 'grant_present': True}, [{ + 'code': 'repair_grant_consumed', 'satisfied': False, + 'missing_or_unknown_fact': 'unused original repair grant', + 'next_action': 'inspect the original repair admission', + }], lifecycle='unknown', + legacy_message='this run already received its one repair grant') spec = self.effective_spec(run_id) finalized = (spec.get("resource_cleanup_version") == 1 and row["cleanup"] == "confirmed") @@ -2016,8 +2029,6 @@ def continue_repair( roles = state.get("roles") checks = state.get("checks") iteration = state.get("iteration") - from .delivery_lifecycle import require_transition - facts = { "operation": "repair_continuation", "owner": "repair_admission", "owner_proof": True, "normalized_lifecycle": "stopped", @@ -2085,22 +2096,47 @@ def continue_repair( for role in roles ) ): - raise ValueError("original implementer session or role cleanup is unconfirmed") + require_observed_predicates('repair_continuation', 'repair_admission', + {'row': dict(row), 'closed': closed}, [{ + 'code': 'repair_role_custody_unknown', 'satisfied': None, + 'missing_or_unknown_fact': 'passed implementation session and confirmed ' + 'role cleanup', + 'next_action': 'inspect the original implementation and role custody', + }], lifecycle='stopped', owner_proof=None, + legacy_message='original implementer session or role cleanup is unconfirmed') for key in ("prepublish", "local", "browser_qa"): result = checks.get(key) if result is None: continue if not isinstance(result, dict) or result.get("state") == "unknown": - raise ValueError("a check has an unknown outcome") + require_observed_predicates('repair_continuation', 'repair_admission', + {'row': dict(row), 'closed': closed}, [{ + 'code': 'repair_check_outcome_unknown', 'satisfied': None, + 'missing_or_unknown_fact': f'known {key} check outcome', + 'next_action': 'inspect the original check outcome', + }], lifecycle='stopped', owner_proof=None, + legacy_message='a check has an unknown outcome') if result.get("cleanup") == "unknown" or ( key == "browser_qa" and result.get("cleanup") != "confirmed" ): - raise ValueError("a check has unconfirmed cleanup") + require_observed_predicates('repair_continuation', 'repair_admission', + {'row': dict(row), 'closed': closed}, [{ + 'code': 'repair_check_cleanup_unknown', 'satisfied': None, + 'missing_or_unknown_fact': f'confirmed {key} check cleanup', + 'next_action': 'reconcile the original check cleanup', + }], lifecycle='stopped', owner_proof=None, + legacy_message='a check has unconfirmed cleanup') if any( not isinstance(item, dict) or item.get("cleanup") != "confirmed" for item in result.get("results", []) ): - raise ValueError("a check child has unconfirmed cleanup") + require_observed_predicates('repair_continuation', 'repair_admission', + {'row': dict(row), 'closed': closed}, [{ + 'code': 'repair_check_child_cleanup_unknown', 'satisfied': None, + 'missing_or_unknown_fact': f'confirmed {key} check child cleanup', + 'next_action': 'reconcile the original check child custody', + }], lifecycle='stopped', owner_proof=None, + legacy_message='a check child has unconfirmed cleanup') findings = failed_gate_diagnostics(state, spec) if not findings: raise ValueError("failed gate supplied no repair diagnostics") @@ -2138,15 +2174,80 @@ def continue_repair( if finalized: recovery.update(original_recovery=previous_recovery, predecessor_spec=spec, finalized_checkpoint=True, cleanup_digest=cleanup_digest) + def observe_admission(db): + """Check the same frozen inventory before preview and under admission custody.""" + current = db.execute("SELECT * FROM delivery_runs WHERE run_id=?", (run_id,)).fetchone() + claim = self.state.claim_for(db, spec["work_id"]) + attempts = db.execute( + "SELECT role,iteration,state,session_id,cleanup " + "FROM delivery_attempts WHERE run_id=?", (run_id,), + ).fetchall() + effects = db.execute( + "SELECT kind,state,observed_json FROM delivery_effects WHERE run_id=?", (run_id,), + ).fetchall() + grant = db.execute( + "SELECT 1 FROM delivery_repair_grants WHERE run_id=?", (run_id,), + ).fetchone() + checkpoint = (current is not None + and current["request_json"] == row["request_json"] + and current["workflow_id"] == row["workflow_id"] + and current["recovery_json"] == row["recovery_json"] + and current["phase"] == "blocked" and current["outcome"] == "blocked" + and current["execution_state"] == "blocked" + and current["cleanup"] in ({"none", "confirmed"} if stopped_claim else {"none"}) + and current["error"] == state["error"] + and current["protocol_revision"] == state["revision"] + and current["iteration"] == iteration + and json.loads(current["candidate_json"] or "null") == candidate + and json.loads(current["pr_json"] or "null") == pr + and json.loads(current["checks_json"] or "{}") == checks) + owner = (claim is None if stopped_claim else + claim is not None and claim["owner"] == f"external:devflow:{run_id}") + attempt_custody = (len(attempts) == len(roles) + and all(item["state"] == "finished" and item["cleanup"] == "confirmed" + for item in attempts) + and sorted((item["role"], item["iteration"], item["session_id"]) + for item in attempts) + == sorted((item["role"], item["iteration"], item["session_id"]) + for item in roles)) + effect_custody = (any(item["kind"] == "publish" for item in effects) + and all(item["state"] == "complete" and item["observed_json"] is not None + for item in effects)) + require_observed_predicates('repair_continuation', 'repair_admission', { + 'row': dict(row), 'current': dict(current) if current else None, 'claim': claim, + 'attempts': [dict(item) for item in attempts], + 'effects': [dict(item) for item in effects], 'grant_present': grant is not None, + }, [ + {'code': 'repair_checkpoint_changed', 'satisfied': checkpoint, + 'missing_or_unknown_fact': 'unchanged stopped run and closed checkpoint', + 'next_action': 'inspect the retained run and closed predecessor'}, + {'code': 'repair_grant_consumed', 'satisfied': grant is None, + 'missing_or_unknown_fact': 'unused original repair grant', + 'next_action': 'inspect the original repair admission'}, + {'code': 'repair_owner_conflict', 'satisfied': owner, + 'missing_or_unknown_fact': 'original or released work claim', + 'next_action': 'inspect the current work owner'}, + {'code': 'repair_attempt_custody_unknown', + 'satisfied': True if attempt_custody else None, + 'missing_or_unknown_fact': 'finished matching attempts and confirmed cleanup', + 'next_action': 'reconcile the original attempt custody'}, + {'code': 'repair_effects_unsettled', + 'satisfied': True if effect_custody else None, + 'missing_or_unknown_fact': 'settled original publication and other effects', + 'next_action': 'reconcile the original effects'}, + ], lifecycle='stopped', owner_proof=( + True if (checkpoint and grant is None and owner and attempt_custody + and effect_custody) else None), + legacy_message='repair continuation lost its frozen run or ownership') + return current + if preflight: from .delivery_policy_recovery import work_binding with self._connect() as db: + db.execute("BEGIN") work_binding(self, spec, db) - claim = self.state.claim_for(db, spec["work_id"]) - if (claim is not None if stopped_claim else - claim is None or claim["owner"] != f"external:devflow:{run_id}"): - raise ValueError("repair preflight claim authority changed") + observe_admission(db) return {**response, "preflight": True, "diagnostics": findings, "lifecycle_assessment": assessment, "title_constraint_sha256": None} if finalized: @@ -2171,66 +2272,13 @@ def continue_repair( with self._connect() as db: db.execute("BEGIN IMMEDIATE") prior = db.execute( - "SELECT request_digest,response_json FROM delivery_commands WHERE command_id=?", - (command_id,), + "SELECT run_id,request_digest,response_json FROM delivery_commands " + "WHERE command_id=?", (command_id,), ).fetchone() if prior: - if prior["request_digest"] != command_digest: - raise ValueError("command ID already belongs to different inputs") + command_receipt('repair_continuation', prior, run_id, command_digest) return json.loads(prior["response_json"]) - current = db.execute("SELECT * FROM delivery_runs WHERE run_id=?", (run_id,)).fetchone() - claim = self.state.claim_for(db, spec["work_id"]) - attempts = db.execute( - "SELECT role,iteration,state,session_id,cleanup " - "FROM delivery_attempts WHERE run_id=?", - (run_id,), - ).fetchall() - effects = db.execute( - "SELECT kind,state,observed_json FROM delivery_effects WHERE run_id=?", - (run_id,), - ).fetchall() - if ( - current is None - or current["request_json"] != row["request_json"] - or current["workflow_id"] != row["workflow_id"] - or current["recovery_json"] != row["recovery_json"] - or current["phase"] != "blocked" - or current["outcome"] != "blocked" - or current["execution_state"] != "blocked" - or current["cleanup"] not in ( - {"none", "confirmed"} if stopped_claim else {"none"} - ) - or current["error"] != state["error"] - or current["protocol_revision"] != state["revision"] - or current["iteration"] != iteration - or json.loads(current["candidate_json"] or "null") != candidate - or json.loads(current["pr_json"] or "null") != pr - or json.loads(current["checks_json"] or "{}") != checks - or db.execute( - "SELECT 1 FROM delivery_repair_grants WHERE run_id=?", (run_id,) - ).fetchone() - or (claim is not None if stopped_claim else - claim is None or claim["owner"] != f"external:devflow:{run_id}") - or len(attempts) != len(roles) - or any( - item["state"] != "finished" or item["cleanup"] != "confirmed" - for item in attempts - ) - or sorted( - (item["role"], item["iteration"], item["session_id"]) - for item in attempts - ) - != sorted( - (item["role"], item["iteration"], item["session_id"]) - for item in roles - ) - or not any(item["kind"] == "publish" for item in effects) - or any( - item["state"] != "complete" or item["observed_json"] is None - for item in effects - ) - ): - raise ValueError("repair continuation lost its frozen run or ownership") + current = observe_admission(db) if stopped_claim: from .delivery_policy_recovery import work_binding diff --git a/runtime/tests/test_delivery_store_lifecycle_240.py b/runtime/tests/test_delivery_store_lifecycle_240.py index d3add79d..3bfba8dd 100644 --- a/runtime/tests/test_delivery_store_lifecycle_240.py +++ b/runtime/tests/test_delivery_store_lifecycle_240.py @@ -37,7 +37,7 @@ async def test_default_repair_public_assessment_and_admission_agree(service, mon from devflow_temporal.delivery_api import create_app store, request = service - _spec, _state, command = _failed_published_repair_fixture(store, request, monkeypatch) + _spec, state, command = _failed_published_repair_fixture(store, request, monkeypatch) app = create_app(store.config.path) app.state.delivery.store = store host = store.config.dashboard_url @@ -56,8 +56,90 @@ async def test_default_repair_public_assessment_and_admission_agree(service, mon actual = await client.post(base + 'continue-repair', json=stale, headers=headers) assert actual.status_code == 409 and actual.json()['detail'] == preview.json() assert preview.json()['blockers'][0]['code'] == 'revision_changed' + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET state='running' WHERE role='implement'") + attempt_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + attempt_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert not attempt_preview.json()['eligible'], (attempt_preview.json(), + attempt_admission.json()) + assert attempt_admission.status_code == 409 + assert attempt_admission.json()['detail'] == attempt_preview.json() + assert 'repair_attempt_custody_unknown' in { + b['code'] for b in attempt_preview.json()['blockers']} + with store._connect() as db: + assert not db.execute('SELECT 1 FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone() + db.execute("UPDATE delivery_attempts SET state='finished' WHERE role='implement'") + db.execute("UPDATE delivery_effects SET state='unknown' WHERE kind='publish'") + effect_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + effect_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert not effect_preview.json()['eligible'] and effect_admission.status_code == 409 + assert effect_admission.json()['detail'] == effect_preview.json() + assert 'repair_effects_unsettled' in { + b['code'] for b in effect_preview.json()['blockers']} + with store._connect() as db: + db.execute("UPDATE delivery_effects SET state='complete' WHERE kind='publish'") + state['roles'][0]['cleanup'] = 'unknown' + custody_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + custody_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert custody_admission.status_code == 409 + assert custody_admission.json()['detail'] == custody_preview.json(), ( + custody_preview.json(), custody_admission.json()) + assert 'repair_role_custody_unknown' in { + b['code'] for b in custody_preview.json()['blockers']} + state['roles'][0]['cleanup'] = 'confirmed' + state['checks']['local'] = {'state': 'unknown'} + check_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + check_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert check_admission.status_code == 409 + assert check_admission.json()['detail'] == check_preview.json() + assert 'repair_check_outcome_unknown' in { + b['code'] for b in check_preview.json()['blockers']} + del state['checks']['local'] accepted = await client.post(base + 'continue-repair', json=command, headers=headers) assert accepted.status_code == 200 and accepted.json()['authorized_through_iteration'] == 4 + replay = await client.post(base + 'continue-repair', json=command, headers=headers) + assert replay.status_code == 200 and replay.json() == accepted.json() + replay_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + assert replay_preview.json()['disposition'] == 'already_admitted' + conflict = {**command, 'expected_revision': command['expected_revision'] + 1} + conflict_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': conflict}) + conflict_admission = await client.post(base + 'continue-repair', json=conflict, + headers=headers) + assert conflict_admission.status_code == 409 + assert conflict_admission.json()['detail'] == conflict_preview.json() + assert conflict_preview.json()['blockers'][0]['code'] == 'command_conflict' + another = {**command, 'command_id': 'another-repair-grant'} + grant_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': another}) + grant_admission = await client.post(base + 'continue-repair', json=another, headers=headers) + assert grant_admission.status_code == 409 + assert grant_admission.json()['detail'] == grant_preview.json() + assert 'repair_grant_consumed' in {b['code'] for b in grant_preview.json()['blockers']} + with store._connect() as db: + assert db.execute('SELECT count(*) FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone()[0] == 1 + assert not db.execute('SELECT 1 FROM delivery_commands WHERE command_id=?', + (another['command_id'],)).fetchone() + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + ('{}', command['command_id'])) + corrupt_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + corrupt_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert corrupt_admission.status_code == 409 + assert corrupt_admission.json()['detail'] == corrupt_preview.json() + assert 'command_receipt_unknown' in {b['code'] for b in corrupt_preview.json()['blockers']} @pytest.mark.asyncio From cc4424332d8c41106180e76c365d7a3294a79887 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:56:45 +0100 Subject: [PATCH 16/24] fix(recovery): recheck historical work binding --- docs/temporal-runtime.md | 4 ++- .../src/devflow_temporal/delivery_store.py | 23 ++++++++++--- .../test_delivery_store_lifecycle_240.py | 33 +++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/docs/temporal-runtime.md b/docs/temporal-runtime.md index 9271cd2a..41451c3e 100644 --- a/docs/temporal-runtime.md +++ b/docs/temporal-runtime.md @@ -254,7 +254,9 @@ The lifecycle vocabulary distinguishes pending, running, stopping, stopped, recovering, terminal and unknown projections. A projection or successful assessment is not a custody grant: the owning admission rechecks the exact candidate, original authority, effects, cleanup and finite budget under its -existing fences. The transition table does not choose a route. Pause states are +existing fences. Historical ordinary repair also checks the original work, issue +and repository binding in both preview and admission. The transition table does +not choose a route. Pause states are reserved for the pause owner; queued pause support does not prove a running workflow has settled. An ordinary preparation failure before plan acceptance cannot use implementation resume without an accepted plan. Inspect the retained diff --git a/runtime/src/devflow_temporal/delivery_store.py b/runtime/src/devflow_temporal/delivery_store.py index e124e7d1..0467b290 100644 --- a/runtime/src/devflow_temporal/delivery_store.py +++ b/runtime/src/devflow_temporal/delivery_store.py @@ -2174,6 +2174,8 @@ def continue_repair( if finalized: recovery.update(original_recovery=previous_recovery, predecessor_spec=spec, finalized_checkpoint=True, cleanup_digest=cleanup_digest) + from .delivery_policy_recovery import work_binding + def observe_admission(db): """Check the same frozen inventory before preview and under admission custody.""" current = db.execute("SELECT * FROM delivery_runs WHERE run_id=?", (run_id,)).fetchone() @@ -2238,15 +2240,28 @@ def observe_admission(db): ], lifecycle='stopped', owner_proof=( True if (checkpoint and grant is None and owner and attempt_custody and effect_custody) else None), - legacy_message='repair continuation lost its frozen run or ownership') + legacy_message=('repair preflight claim authority changed' + if preflight and not owner + else 'repair continuation lost its frozen run or ownership')) + try: + work_binding(self, spec, db) + except ValueError as exc: + from .delivery_lifecycle import RecoveryRefused + + if isinstance(exc, RecoveryRefused): + raise + require_observed_predicates('repair_continuation', 'repair_admission', { + 'row': dict(row), 'current': dict(current), 'claim': claim, + }, [{ + 'code': 'repair_work_binding_unproven', 'satisfied': None, + 'missing_or_unknown_fact': 'original work, issue and repository binding', + 'next_action': 'inspect the original work and repository authority', + }], lifecycle='stopped', owner_proof=None, legacy_message=str(exc)) return current if preflight: - from .delivery_policy_recovery import work_binding - with self._connect() as db: db.execute("BEGIN") - work_binding(self, spec, db) observe_admission(db) return {**response, "preflight": True, "diagnostics": findings, "lifecycle_assessment": assessment, "title_constraint_sha256": None} diff --git a/runtime/tests/test_delivery_store_lifecycle_240.py b/runtime/tests/test_delivery_store_lifecycle_240.py index 3bfba8dd..30e20c49 100644 --- a/runtime/tests/test_delivery_store_lifecycle_240.py +++ b/runtime/tests/test_delivery_store_lifecycle_240.py @@ -57,6 +57,39 @@ async def test_default_repair_public_assessment_and_admission_agree(service, mon assert actual.status_code == 409 and actual.json()['detail'] == preview.json() assert preview.json()['blockers'][0]['code'] == 'revision_changed' with store._connect() as db: + original_owner = db.execute('SELECT owner FROM claims WHERE work_id=?', + (request['work_id'],)).fetchone()[0] + original_repository = db.execute('SELECT repository FROM works WHERE id=?', + (request['work_id'],)).fetchone()[0] + db.execute("UPDATE claims SET owner='external:devflow:another' WHERE work_id=?", + (request['work_id'],)) + owner_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + owner_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert owner_admission.status_code == 409 + assert owner_admission.json()['detail'] == owner_preview.json() + assert 'repair_owner_conflict' in {b['code'] for b in owner_preview.json()['blockers']} + with store._connect() as db: + assert not db.execute('SELECT 1 FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone() + db.execute('UPDATE claims SET owner=? WHERE work_id=?', + (original_owner, request['work_id'])) + db.execute("UPDATE works SET repository='github.com/example/other' WHERE id=?", + (request['work_id'],)) + work_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + work_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert work_admission.status_code == 409 + assert work_admission.json()['detail'] == work_preview.json() + assert 'repair_work_binding_unproven' in { + b['code'] for b in work_preview.json()['blockers']} + with store._connect() as db: + assert not db.execute('SELECT 1 FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone() + db.execute('UPDATE works SET repository=? WHERE id=?', + (original_repository, request['work_id'])) db.execute("UPDATE delivery_attempts SET state='running' WHERE role='implement'") attempt_preview = await client.post(base + 'assess-transition', json={ 'kind': 'repair_continuation', 'request': command}) From adb8743a73419b75646fffecae74e7997a61cb1c Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 20:49:40 +0100 Subject: [PATCH 17/24] fix(recovery): back off retained readiness retries --- docs/trusted-local-delivery.md | 6 ++ .../devflow_temporal/delivery_gate_retry.py | 23 +++++- .../delivery_readiness_gate_retry.py | 25 +++++++ ...test_delivery_feature_readiness_handoff.py | 1 + .../test_delivery_readiness_gate_retry.py | 73 ++++++++++++++++++- 5 files changed, 125 insertions(+), 3 deletions(-) diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index ef1c0e7e..7ae0df6f 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -188,6 +188,12 @@ for the unchanged candidate, preserving the old receipt, attempt and effect ledger. Every retry probes readiness again and runs the original frozen gate; its readiness receipt is never a gate pass. Other failed gates, pending product findings, uncertain cleanup, changed source or unsettled effects refuse admission. +The first readiness retry waits 30 seconds after the authenticated predecessor +closes; the second waits 60 seconds after its predecessor closes. A missing or +ambiguous close time is reported as `readiness_retry_clock_unknown`, and an +unexpired delay as `readiness_retry_backoff`. Waiting allocates no new attempt, +admission, effect or product-repair grant. The delay is derived from the closed +workflow, so restarting the control service does not restart or bypass it. The initial route is qualified only for the fake fixture provider: native runs refuse with `native_descendant_settlement_unknown` until a complete descendant settlement contract can be authenticated. An exhausted retry allowance reports diff --git a/runtime/src/devflow_temporal/delivery_gate_retry.py b/runtime/src/devflow_temporal/delivery_gate_retry.py index 21d948f5..4df76f9c 100644 --- a/runtime/src/devflow_temporal/delivery_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_gate_retry.py @@ -897,9 +897,17 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= else historical_teardown or prelaunch_observation or ( _stopped_cleanup(spec, read_only=True) if _observation else _stopped_cleanup(spec))) + readiness_backoff = None if readiness_retry: from .delivery_lifecycle import require_transition - from .delivery_readiness_gate_retry import RETRY_LIMIT + from .delivery_readiness_gate_retry import RETRY_LIMIT, retry_backoff + + remaining = readiness_generations < RETRY_LIMIT + backoff_elapsed = None + if remaining: + readiness_backoff, backoff_elapsed = retry_backoff( + closed.get('closed_at'), readiness_generations + 1) + clock_known = not remaining or backoff_elapsed is not None # An observed native monitor does not prove its complete descendant set. # The fake fixture has no native actor; real execution needs a separate @@ -909,6 +917,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= 'normalized_lifecycle': 'stopped', 'proof_digest': digest({'row': _sealed_row(row), 'cleanup': cleanup, 'readiness': readiness_observation, + 'readiness_backoff': readiness_backoff, 'prior_readiness_retries': readiness_generations}), 'comparisons': [], 'predicates': [ {'code': 'native_descendant_settlement_unknown', @@ -916,9 +925,18 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= 'missing_or_unknown_fact': 'complete native descendant settlement', 'next_action': 'obtain authenticated native descendant settlement'}, {'code': 'readiness_retry_limit_exhausted', - 'satisfied': readiness_generations < RETRY_LIMIT, + 'satisfied': remaining, 'missing_or_unknown_fact': 'remaining infrastructure retry allowance', 'next_action': 'inspect retained failures and reconcile the environment'}, + {'code': 'readiness_retry_clock_unknown', + 'satisfied': True if clock_known else None, + 'missing_or_unknown_fact': 'authenticated predecessor close time', + 'next_action': 'inspect the retained workflow closure'}, + {'code': 'readiness_retry_backoff', + 'satisfied': backoff_elapsed if remaining and clock_known else True, + 'missing_or_unknown_fact': 'elapsed infrastructure retry backoff', + 'next_action': ('retry after ' + readiness_backoff['retry_not_before'] + if remaining and clock_known else 'inspect the retry allowance')}, ], }, {}) result = {'row': _sealed_row(row), 'closed': closed, 'original_spec': spec, @@ -932,6 +950,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= **({'ordinary_prerequisite_observation': prerequisite_observation} if prerequisite_observation else {}), **({'readiness_observation': readiness_observation} if readiness_observation else {}), + **({'readiness_backoff': readiness_backoff} if readiness_backoff else {}), **({'report_observation': report_observation} if report_observation else {}), **({'verification_observation': verification_observation} if verification_observation else {}), diff --git a/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py b/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py index 47cd5ca0..886a7197 100644 --- a/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py @@ -8,15 +8,40 @@ import hashlib import json import re +from datetime import UTC, datetime, timedelta from pathlib import Path from .contracts import digest from .delivery_runtime_location import private_bytes RETRY_LIMIT = 2 +BACKOFF_SECONDS = 30 SLOTS = {"prepublish": 100, "local": 101, "browser_qa": 102} +def _utc_now(): + return datetime.now(UTC) + + +def retry_backoff(closed_at, generation): + """Bind the next retry's delay to the authentic predecessor closure.""" + if type(generation) is not int or not 1 <= generation <= RETRY_LIMIT: + raise ValueError("readiness retry generation exceeds its finite bound") + delay = BACKOFF_SECONDS * 2 ** (generation - 1) + proof = {"closed_at": closed_at, "delay_seconds": delay, "retry_not_before": None} + if not isinstance(closed_at, str): + return proof, None + try: + stopped = datetime.fromisoformat(closed_at) + if stopped.tzinfo is None or stopped.utcoffset() is None: + return proof, None + retry_at = stopped.astimezone(UTC) + timedelta(seconds=delay) + except (ValueError, OverflowError): + return proof, None + proof["retry_not_before"] = retry_at.isoformat() + return proof, _utc_now() >= retry_at + + def probe_attempt(spec, candidate, slot): """Reserve disjoint readiness evidence names for the two retry generations.""" if slot not in SLOTS.values(): diff --git a/runtime/tests/test_delivery_feature_readiness_handoff.py b/runtime/tests/test_delivery_feature_readiness_handoff.py index 37a300aa..ef8420a6 100644 --- a/runtime/tests/test_delivery_feature_readiness_handoff.py +++ b/runtime/tests/test_delivery_feature_readiness_handoff.py @@ -161,6 +161,7 @@ def admit(config, *args, **kwargs): closed = { "workflow_id": row["workflow_id"], "execution_run_id": "closed-worker", + "closed_at": "2020-01-01T00:00:00+00:00", "request_digest": child["request_digest"], "recovery_digest": None, "result": state, diff --git a/runtime/tests/test_delivery_readiness_gate_retry.py b/runtime/tests/test_delivery_readiness_gate_retry.py index 83c12b40..9ee46d01 100644 --- a/runtime/tests/test_delivery_readiness_gate_retry.py +++ b/runtime/tests/test_delivery_readiness_gate_retry.py @@ -10,6 +10,7 @@ import json from contextlib import asynccontextmanager from copy import deepcopy +from datetime import UTC, datetime, timedelta from pathlib import Path import httpx @@ -169,6 +170,7 @@ def failure(execution, current): closed = { "workflow_id": "delivery-run-1", "execution_run_id": "closed-original", + "closed_at": "2020-01-01T00:00:00+00:00", "request_digest": spec["request_digest"], "recovery_digest": None, "result": state, @@ -194,8 +196,15 @@ async def client_for(store): @pytest.mark.asyncio -async def test_public_retry_preserves_receipts_and_bounds_repeated_failures(stopped_readiness): +async def test_public_retry_preserves_receipts_and_bounds_repeated_failures( + stopped_readiness, monkeypatch +): store, broker, state, command, original, closed, failure = stopped_readiness + from devflow_temporal import delivery_readiness_gate_retry + + clock = [datetime(2020, 1, 1, tzinfo=UTC)] + monkeypatch.setattr(delivery_readiness_gate_retry, "_utc_now", lambda: clock[0], raising=False) + closed["closed_at"] = clock[0].isoformat() candidate = deepcopy(state["candidate"]) retained = Path(original["readiness"]["receipt"]).read_bytes() with store._connect() as db: @@ -203,7 +212,64 @@ async def test_public_retry_preserves_receipts_and_bounds_repeated_failures(stop initial_effects = [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] attempts = [original["readiness"]["attempt"]] async with client_for(store) as (client, headers): + # A missing close time cannot grant a retry or silently disable backoff. + for invalid_close in (None, "2020-01-01T00:00:00"): + closed["closed_at"] = invalid_close + assessment = await client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert assessment.status_code == 200 + assert any(item["code"] == "readiness_retry_clock_unknown" + and item["satisfied"] is None + for item in assessment.json()["blockers"]) + closed["closed_at"] = clock[0].isoformat() for generation in (1, 2): + # Infrastructure retries wait 30 seconds and then 60 seconds; + # no product allowance or attempt is allocated during the wait. + waiting = await client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert waiting.status_code == 200 + assert any(item["code"] == "readiness_retry_backoff" + for item in waiting.json()["blockers"]) + if generation == 2: + # Restarting the control service must not reset the delay. + restarted_store = DeliveryStore(DeliveryConfig.load(store.config.path)) + async with client_for(restarted_store) as (restarted_client, _): + after_restart = await restarted_client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert after_restart.status_code == 200 + assert any(item["code"] == "readiness_retry_backoff" + for item in after_restart.json()["blockers"]) + with store._connect() as db: + admissions = [dict(x) for x in db.execute("SELECT * FROM delivery_gate_admissions")] + denied = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert denied.status_code == 409 + with store._connect() as db: + assert [dict(x) for x in db.execute("SELECT * FROM delivery_attempts")] == ( + initial_attempts + ) + assert [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] == ( + initial_effects + ) + assert [dict(x) for x in db.execute("SELECT * FROM delivery_gate_admissions")] == ( + admissions + ) + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + delay = 30 * 2 ** (generation - 1) + clock[0] += timedelta(seconds=delay - 1) + before_deadline = await client.post( + "/api/runs/run-1/repair-admission-preflight", json=command + ) + assert before_deadline.status_code == 409 + assert "readiness_retry_backoff" in before_deadline.text + clock[0] += timedelta(seconds=1) preflight = await client.post( "/api/runs/run-1/repair-admission-preflight", json=command ) @@ -237,6 +303,10 @@ async def test_public_retry_preserves_receipts_and_bounds_repeated_failures(stop assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() assert execution["gate_retry_generation"] == generation assert execution["gate_retry_stage"] == "readiness" + assert recovery["seal"]["readiness_backoff"]["delay_seconds"] == delay + assert recovery["seal"]["readiness_backoff"]["retry_not_before"] == ( + clock[0].isoformat() + ) assert recovery["state"]["checks"]["prepublish"]["readiness"]["state"] == "failed" assert readback(store, execution, recovery) is None if generation == 1: @@ -276,6 +346,7 @@ async def test_public_retry_preserves_receipts_and_bounds_repeated_failures(stop closed.update( workflow_id=admitted["workflow_id"], execution_run_id=f"closed-{generation}", + closed_at=clock[0].isoformat(), recovery_digest=digest(recovery), result=current, ) From efb7a5dc51f1a9f670f54fea00d2c030533bff22 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 20:56:43 +0100 Subject: [PATCH 18/24] fix(recovery): preserve fake provider continuation --- .../devflow_temporal/delivery_checkpoints.py | 6 +- .../devflow_temporal/delivery_continuation.py | 10 +- runtime/src/devflow_temporal/supervisor.py | 5 + .../test_delivery_checkpoint_fake_temporal.py | 228 ++++++++++++++++++ 4 files changed, 246 insertions(+), 3 deletions(-) create mode 100644 runtime/tests/test_delivery_checkpoint_fake_temporal.py diff --git a/runtime/src/devflow_temporal/delivery_checkpoints.py b/runtime/src/devflow_temporal/delivery_checkpoints.py index 324e4b75..caebe3b2 100644 --- a/runtime/src/devflow_temporal/delivery_checkpoints.py +++ b/runtime/src/devflow_temporal/delivery_checkpoints.py @@ -875,7 +875,11 @@ def references(value): if not home.is_relative_to(state): raise ValueError("checkpoint session home escaped its original run") _openers(home) - paths = session_state_paths(home, attempt["session_id"]) + # The fixed fake provider has a synthetic session ID but writes no + # transcript. Preserve fixture transcripts when present; native + # providers must always supply their required session. + paths = session_state_paths( + home, attempt["session_id"], allow_absent=spec["provider"] == "fake") selected_manifest(home, paths) # Private owned regular files, no linked ancestors. selected.update((home / item).relative_to(state).as_posix() for item in paths) elif attempt["session_id"] and spec["provider"] != "fake": diff --git a/runtime/src/devflow_temporal/delivery_continuation.py b/runtime/src/devflow_temporal/delivery_continuation.py index 6c225762..a856325a 100644 --- a/runtime/src/devflow_temporal/delivery_continuation.py +++ b/runtime/src/devflow_temporal/delivery_continuation.py @@ -67,14 +67,18 @@ def selected_digest(root: Path, paths: list[str]) -> str: ) -def session_state_paths(role_home: Path, session_id: str) -> list[str]: - """Select only provider session data; auth, profile, caches and tools are regenerated.""" +def session_state_paths( + role_home: Path, session_id: str, *, allow_absent: bool = False, +) -> list[str]: + """Select session data; allow absence only for fixed providers without transcripts.""" home_info = role_home.lstat() if not stat.S_ISDIR(home_info.st_mode) or home_info.st_uid != os.getuid(): raise ValueError("continuation role home is not an owned directory") codex = role_home / "codex" if not codex.is_dir() or codex.is_symlink(): + if allow_absent and not os.path.lexists(codex): + return [] raise ValueError("continuation Codex home is unavailable") paths = [ "codex/" + file.name @@ -89,6 +93,8 @@ def session_state_paths(role_home: Path, session_id: str) -> list[str]: if session_id in file.name and (file.is_file() or file.is_symlink()) ) if not any(session_id in Path(path).name for path in paths): + if allow_absent: + return [] raise ValueError("continuation session transcript is missing") return sorted(paths) diff --git a/runtime/src/devflow_temporal/supervisor.py b/runtime/src/devflow_temporal/supervisor.py index d8677a8d..9bf0ad76 100644 --- a/runtime/src/devflow_temporal/supervisor.py +++ b/runtime/src/devflow_temporal/supervisor.py @@ -550,6 +550,11 @@ async def _run(self, request: dict[str, Any]) -> dict[str, Any]: from .delivery_resources import read_private, write_private output_candidate = await writer_thread(spec, self._completed_candidate, request, result) + if spec.get("provider") == "fake": + # This fixed fixture role cannot launch candidate-controlled children. + # Bind the supervisor's joined-process observation to its result. + result.setdefault("cleanup", "confirmed") + write_private(result_path, result) metadata = read_private(start_path) metadata.update(output_candidate=output_candidate, result_digest=digest(result)) write_private(start_path, metadata) diff --git a/runtime/tests/test_delivery_checkpoint_fake_temporal.py b/runtime/tests/test_delivery_checkpoint_fake_temporal.py new file mode 100644 index 00000000..c566d91b --- /dev/null +++ b/runtime/tests/test_delivery_checkpoint_fake_temporal.py @@ -0,0 +1,228 @@ +"""Public recovery through a real Temporal controller and the fixed fake provider.""" + +from __future__ import annotations + +import asyncio +import json +import shutil +from contextlib import suppress +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest +from temporal_test_server import local_temporal +from temporalio import activity +from temporalio.worker import UnsandboxedWorkflowRunner, Worker +from test_delivery_store import service as service + +from devflow_temporal import delivery_activities as activities +from devflow_temporal.delivery_api import _dispatch_start, create_app +from devflow_temporal.delivery_broker import DeliveryBroker +from devflow_temporal.delivery_config import DeliveryConfig +from devflow_temporal.delivery_continuation import session_state_paths +from devflow_temporal.delivery_resource_maintenance import prune +from devflow_temporal.delivery_store import DeliveryStore +from devflow_temporal.delivery_workflow import DeliveryWorkflow + + +@pytest.mark.asyncio +async def test_public_recovery_resumes_real_fake_implementation(service, monkeypatch): + """Catch a false resumable ACK when the real fake role cannot be continued.""" + async with local_temporal() as environment: + original, request = service + config = json.loads(original.config.path.read_text()) + config["temporal_address"] = environment.client.service_client.config.target_host + config["workspace_custody_version"] = 1 + config["repositories"]["fixture"]["allowed_paths"].append("devflow-fake-change.txt") + original.config.path.write_text(json.dumps(config)) + store = DeliveryStore(DeliveryConfig.load(original.config.path)) + # Fake admissions normally omit terminal resource cleanup. Enable the real + # finalization activity in this fixture so it covers the retention contract. + admitted_spec = store.config.admit(request) + admitted_spec["resource_cleanup_version"] = 1 + with monkeypatch.context() as admission: + admission.setattr(DeliveryConfig, "admit", lambda *_args, **_kwargs: admitted_spec) + store.submit(request) + spec = store.effective_spec(request["run_id"]) + broker = DeliveryBroker(store, spec) + calls = [] + + @activity.defn(name="delivery_precheck") + async def stopped_gate(body): + calls.append(("stopped-gate", body["iteration"])) + return {"state": "failed", "source_unchanged": True, + "candidate_id": body["candidate"]["id"], "results": [{ + "id": "fixture-prerequisite", "passed": False, + "failure_kind": "preparation", "launched": False, + "cleanup": "confirmed"}]} + + @activity.defn(name="delivery_tracker_start") + async def tracker_start(_body): + return {"state": "consistent"} + + service_context = SimpleNamespace(store=store, config=store.config) + async with Worker( + environment.client, task_queue=store.config.queue, + workflow_runner=UnsandboxedWorkflowRunner(), workflows=[DeliveryWorkflow], + activities=[activities.delivery_project, activities.delivery_prepare, + activities.delivery_role, activities.delivery_finalize_resources, + stopped_gate, tracker_start], + interceptors=[activities.DatabaseContentionInterceptor()], + ): + await _dispatch_start( + service_context, environment.client, store.pending_starts()[0], spec) + original_handle = environment.client.get_workflow_handle("delivery-" + spec["run_id"]) + try: + stopped = await asyncio.wait_for(original_handle.result(), 30) + except BaseException: + with suppress(Exception): + await original_handle.terminate("owned fixture cleanup") + raise + + assert stopped["outcome"] == "blocked" and stopped["cleanup"] == "confirmed" + assert stopped["checks"]["prepublish"]["state"] == "failed" + assert calls == [("stopped-gate", 0)] + implementation = stopped["roles"][0] + assert implementation["status"] == "pass" and implementation["cleanup"] == "confirmed" + session_id = implementation["session_id"] + assert session_id == "fake:" + spec["run_id"] + ":implement" + candidate = stopped["candidate"] + home = Path(spec["state_dir"]) / "role-homes/implement" + (home / "codex/sessions").mkdir(parents=True, mode=0o700, exist_ok=True) + assert not list((home / "codex/sessions").rglob("*")) + # The normal provider-session contract still refuses missing transcripts. + with pytest.raises(ValueError, match="continuation session transcript is missing"): + session_state_paths(home, session_id) + with store._connect() as db: + attempt = db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND role='implement'", + (spec["run_id"],)).fetchone() + assert attempt["state"] == "finished" and attempt["cleanup"] == "confirmed" + assert json.loads(attempt["result_json"])["cleanup"] == "confirmed" + assert json.loads(Path(attempt["result_path"]).read_text()) == json.loads( + attempt["result_json"]) + # The tracker boundary is a fixture; emulate its terminal claim release. + store.state.release_work(db, spec["work_id"], "external:devflow:" + spec["run_id"]) + row = dict(db.execute("SELECT * FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone()) + + # No actual GitHub operation is made by the fixture's tracker or publisher. + monkeypatch.setattr(DeliveryStore, "_ensure_no_remote_pr", lambda *_args: None) + app = create_app(store.config.path) + app.state.delivery.store = store + url = store.config.dashboard_url.rstrip("/") + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url=url, + ) as client: + headers = {"origin": url, "x-devflow-csrf": (await client.get("/api/session")).json()[ + "csrf_token"]} + path = "/api/runs/" + spec["run_id"] + "/checkpoints" + saved = await client.post(path, headers=headers, json={ + "command_id": "fake-temporal-save", "expected_revision": row["revision"]}) + assert saved.status_code == 200, saved.text + assert saved.json()["resumable"] + pruned = prune(store, spec["run_id"]) + assert any(root["path"] == str(broker.checkout) and root["state"] == "retained" + for root in pruned["roots"]) + retained = await client.get(path + "/" + saved.json()["checkpoint_id"]) + assert retained.status_code == 200 and retained.json()["resumable"] + shutil.rmtree(Path(spec["source_path"])) + shutil.rmtree(broker.checkout) + if home.exists(): + shutil.rmtree(home) + restored = await client.post( + path + "/" + saved.json()["checkpoint_id"] + "/restore", headers=headers, + json={"command_id": "fake-temporal-restore", "expected_revision": row["revision"]}) + assert restored.status_code == 200, restored.text + assert restored.json()["execution_started"] is False + assert broker.candidate() == candidate + assert (broker.checkout / "devflow-fake-change.txt").read_text() == ( + "Deterministic fake change 0\n") + command = {"continuation_kind": "stopped_delivery_resume", + "command_id": "fake-temporal-resume", + "expected_revision": stopped["revision"], + "expected_iteration": stopped["iteration"], + "expected_candidate_id": candidate["id"], + "expected_candidate_head": candidate["head"], "additional_iterations": 1} + response = await client.post("/api/runs/" + spec["run_id"] + "/continue-repair", + headers=headers, json=command) + assert response.status_code == 200, response.text + assert response.json()["run_id"] == spec["run_id"] + assert response.json()["authorized_through_iteration"] == 1 + replay = await client.post("/api/runs/" + spec["run_id"] + "/continue-repair", + headers=headers, json=command) + assert replay.status_code == 200 and replay.json() == response.json() + + with store._connect() as db: + current = db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id=?", + (spec["run_id"],)).fetchone() + recovery = json.loads(current["recovery_json"]) + assert recovery["candidate"] == candidate and recovery["session_id"] == session_id + assert recovery["state"]["checks"] == stopped["checks"] + assert recovery["state"]["usage"] == stopped["usage"] + assert db.execute("SELECT count(*) FROM delivery_runs").fetchone()[0] == 1 + assert store.state.claim_for(db, spec["work_id"])["owner"] == ( + "external:devflow:" + spec["run_id"]) + assert not db.execute("SELECT 1 FROM delivery_effects WHERE kind='publish'").fetchone() + + @activity.defn(name="delivery_role") + async def role(body): + if body["role"] == "implement": + assert body["resume_session"] == session_id + assert body["candidate"] == candidate + calls.append(("continued-implementation", body["iteration"])) + return await activities.delivery_role(body) + return {"status": "pass", "role": body["role"], "iteration": body["iteration"], + "session_id": "fixture-" + body["role"], "candidate": body["candidate"], + "cleanup": "confirmed", "findings": []} + + @activity.defn(name="delivery_precheck") + async def passing_gate(body): + calls.append(("passing-gate", body["iteration"])) + return {"state": "passed", "candidate_id": body["candidate"]["id"]} + + @activity.defn(name="delivery_publish") + async def publish(body): + calls.append(("fixture-publication", body["iteration"])) + return {"number": 7, "head": body["candidate"]["head"], + "candidate": body["candidate"], "state": "OPEN"} + + @activity.defn(name="delivery_checks") + async def checks(body): + return {"state": "passed", "candidate_id": body["candidate"]["id"]} + + @activity.defn(name="delivery_ci") + async def ci(body): + return {"state": "passed", "head": body["pull_request"]["head"]} + + @activity.defn(name="delivery_tracker") + async def tracker(_body): + return {"state": "consistent"} + + async with Worker( + environment.client, task_queue=store.config.queue, + workflow_runner=UnsandboxedWorkflowRunner(), workflows=[DeliveryWorkflow], + activities=[activities.delivery_project, activities.delivery_repair_preflight, + activities.delivery_finalize_resources, role, passing_gate, publish, + checks, ci, tracker_start, tracker], + interceptors=[activities.DatabaseContentionInterceptor()], + ): + await _dispatch_start(service_context, environment.client, store.pending_starts()[0], + store.effective_spec(spec["run_id"])) + successor = environment.client.get_workflow_handle(response.json()["workflow_id"]) + try: + result = await asyncio.wait_for(successor.result(), 30) + except BaseException: + with suppress(Exception): + await successor.terminate("owned fixture cleanup") + raise + + assert result["outcome"] == "delivered" # Fixture publication and gates only. + assert result["roles"][0] == implementation + assert result["roles"][1]["session_id"] == session_id + assert result["usage"]["implement:0"] == stopped["usage"]["implement:0"] + assert calls == [("stopped-gate", 0), ("continued-implementation", 1), + ("passing-gate", 1), ("fixture-publication", 1)] + assert (broker.checkout / "devflow-fake-change.txt").read_text() == ( + "Deterministic fake change 1\n") From 64f0ad39313e9c9849aae8753c47dc74f53f0604 Mon Sep 17 00:00:00 2001 From: Eloi Date: Sun, 1 Nov 2026 18:27:57 +0100 Subject: [PATCH 19/24] fix(recovery): validate retained evidence on readback --- docs/durable-checkpoints.md | 5 +- .../devflow_temporal/delivery_checkpoints.py | 10 ++++ .../delivery_retained_checkpoint.py | 15 +++--- runtime/tests/test_delivery_checkpoints.py | 48 ++++++++++++++++++- 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/docs/durable-checkpoints.md b/docs/durable-checkpoints.md index 3d46578f..cefd64cb 100644 --- a/docs/durable-checkpoints.md +++ b/docs/durable-checkpoints.md @@ -97,8 +97,9 @@ Readback preserves the original `acknowledged_resumable` value as history. Its current `resumable` observation also compares the admitted authority, run and cumulative budget, issue owner, independent Git owner and candidate branch, original resource manifest, and the current writer witness. If the checkout is -still present, it compares its retained source and index under that witness; -an absent checkout can still be restored from the sealed artifacts. Changed or +still present, it compares its retained source and index under that witness. +Existing evidence and session files must match their sealed state; missing files +and an absent checkout can still be restored from the sealed artifacts. Changed or unavailable proof appears in `resume_blockers`. This is a read-only observation, not a reservation or permission to resume: restoration must reacquire custody and repeat its own admission checks. diff --git a/runtime/src/devflow_temporal/delivery_checkpoints.py b/runtime/src/devflow_temporal/delivery_checkpoints.py index caebe3b2..a19d2e59 100644 --- a/runtime/src/devflow_temporal/delivery_checkpoints.py +++ b/runtime/src/devflow_temporal/delivery_checkpoints.py @@ -1494,6 +1494,7 @@ def read_checkpoint(store, run_id, checkpoint_id): if blocker not in blockers: blockers.append(blocker) retained_source_unknown = False + retained_state_unknown = False try: registry = RunResources(spec, read_only=True) with registry.locked(read_only=True) as resources: @@ -1511,6 +1512,13 @@ def read_checkpoint(store, run_id, checkpoint_id): # The exclusive witness is observational and grants no later work. with store._checkpoint_writer_context(spec): writer_unknown = False + try: + from .delivery_retained_checkpoint import verify_retained_state + + verify_retained_state( + spec, value, _root(spec) / checkpoint_id, allow_missing=True) + except (OSError, ValueError, KeyError): + retained_state_unknown = True checkout = Path(spec["checkout"]) if os.path.lexists(checkout) and _index_baseline_supported(value): try: @@ -1536,6 +1544,8 @@ def read_checkpoint(store, run_id, checkpoint_id): blockers.append({"scope": "writer", "reason": "checkpoint_writer_custody_unknown"}) if retained_source_unknown: blockers.append({"scope": "source", "reason": "checkpoint_retained_source_unverified"}) + if retained_state_unknown: + blockers.append({"scope": "state", "reason": "checkpoint_retained_state_unverified"}) if feature_unknown: blockers.append({"scope": "feature", "reason": feature_unknown}) index_unknown = not _index_baseline_supported(value) diff --git a/runtime/src/devflow_temporal/delivery_retained_checkpoint.py b/runtime/src/devflow_temporal/delivery_retained_checkpoint.py index a8f31a91..4757edde 100644 --- a/runtime/src/devflow_temporal/delivery_retained_checkpoint.py +++ b/runtime/src/devflow_temporal/delivery_retained_checkpoint.py @@ -287,15 +287,18 @@ def _verify_recorded_source(spec, value, checkpoint_root, resources): "branch": branch, "head": head, "index_tree": value["source"]["index_tree"]} -def verify_retained_state(spec, value, checkpoint_root): - """Verify every declared evidence/session byte and authenticated absence.""" +def verify_retained_state(spec, value, checkpoint_root, *, allow_missing=False): + """Verify state targets; missing files can be restored when explicitly allowed.""" for relative, item in value["state_refs"].items(): target = _path(Path(spec["state_dir"]), relative) + try: + info = target.lstat() + except FileNotFoundError: + if item.get("kind") == "absent" or allow_missing: + continue + raise if item.get("kind") == "absent": - if os.path.lexists(target): - raise ValueError("retained checkpoint no-GO absence changed: " + relative) - continue - info = target.lstat() + raise ValueError("retained checkpoint no-GO absence changed: " + relative) if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1: raise ValueError("retained checkpoint evidence is not an owned file: " + relative) raw = target.read_bytes() diff --git a/runtime/tests/test_delivery_checkpoints.py b/runtime/tests/test_delivery_checkpoints.py index 13c7d8e9..fa12c51d 100644 --- a/runtime/tests/test_delivery_checkpoints.py +++ b/runtime/tests/test_delivery_checkpoints.py @@ -582,13 +582,59 @@ async def test_public_checkpoint_routes_require_csrf_and_return_verified_readbac response = await client.post(path, json=command, headers=headers) assert response.status_code == 200, response.text receipt = response.json() - assert (await client.get(path + "/" + receipt["checkpoint_id"])).json()["resumable"] + checkpoint_url = path + "/" + receipt["checkpoint_id"] + assert (await client.get(checkpoint_url)).json()["resumable"] + # Existing state targets must still match the acknowledged evidence. + # Missing targets remain eligible because restore can recreate them. + for target in (evidence, transcript): + original = target.read_bytes() + original_stat = target.stat() + target.write_text("changed retained fixture\n") + observed = (await client.get(checkpoint_url)).json() + assert observed["acknowledged_resumable"] and not observed["resumable"] + assert "checkpoint_retained_state_unverified" in { + blocker["reason"] for blocker in observed["resume_blockers"]} + verified = await client.post( + checkpoint_url + "/verify-retained", headers=headers, + json={"command_id": "verify-changed-" + target.name, + "expected_revision": row["revision"]}) + assert verified.status_code == 409 + assert target.read_text() == "changed retained fixture\n" + target.write_bytes(original) + os.utime(target, ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns)) + # An unobservable parent is not proof of an absent, restorable file. + evidence.parent.chmod(0) + try: + unreadable = (await client.get(checkpoint_url)).json() + assert not unreadable["resumable"] + assert "checkpoint_retained_state_unverified" in { + blocker["reason"] for blocker in unreadable["resume_blockers"]} + finally: + evidence.parent.chmod(0o700) + # A foreign target must not be followed or altered by observation. + original = evidence.read_bytes() + original_stat = evidence.stat() + other = state / "owned-sentinel.txt" + other.write_text("untouched fixture\n") + evidence.unlink() + evidence.symlink_to(other) + linked = (await client.get(checkpoint_url)).json() + assert not linked["resumable"] + assert "checkpoint_retained_state_unverified" in { + blocker["reason"] for blocker in linked["resume_blockers"]} + assert other.read_text() == "untouched fixture\n" + evidence.unlink() + evidence.write_bytes(original) + evidence.chmod(0o600) + os.utime(evidence, ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns)) + assert (await client.get(checkpoint_url)).json()["resumable"] checkpoint = state / "checkpoints" / receipt["checkpoint_id"] assert not list(checkpoint.rglob("auth.json")) shutil.rmtree(Path(spec["source_path"])) shutil.rmtree(broker.checkout) shutil.rmtree(home) evidence.unlink() + assert (await client.get(checkpoint_url)).json()["resumable"] response = await client.post( path + "/" + receipt["checkpoint_id"] + "/restore", json={"command_id": "http-restore", "expected_revision": row["revision"]}, From 8121ac2981c00fd20636300126562bd7feb1add3 Mon Sep 17 00:00:00 2001 From: Eloi Date: Sun, 1 Nov 2026 18:39:32 +0100 Subject: [PATCH 20/24] fix(recovery): validate historical admission receipts --- docs/temporal-runtime.md | 14 ++- .../devflow_temporal/delivery_gate_retry.py | 27 +++-- .../devflow_temporal/delivery_lifecycle.py | 110 ++++++++++++++++-- .../delivery_pending_publication.py | 22 ++-- .../delivery_stopped_resume.py | 27 +++-- .../src/devflow_temporal/delivery_store.py | 7 +- .../test_delivery_feature_resumed_evidence.py | 7 ++ runtime/tests/test_delivery_gate_retry.py | 85 ++++++++++++-- .../test_delivery_pending_publication.py | 39 +++++-- .../test_delivery_store_lifecycle_240.py | 36 ++++-- .../test_delivery_unpublished_cleanup.py | 23 ++-- 11 files changed, 314 insertions(+), 83 deletions(-) diff --git a/docs/temporal-runtime.md b/docs/temporal-runtime.md index 41451c3e..acf5b51c 100644 --- a/docs/temporal-runtime.md +++ b/docs/temporal-runtime.md @@ -268,9 +268,17 @@ command has already been admitted, assessment reports `already_admitted` rather than presenting the old receipt as a fresh grant. Replaying the same command returns its original receipt; reusing its ID with different inputs reports `command_conflict` and queues -no new work. An unverified stored receipt remains unknown. Pending publication also -reports unknown native resource custody when the original process, root, or lease -cannot be proven settled; the existing observer still refuses admission. +no new work. A missing required receipt field, mismatched workflow, or unsupported +retained admission is unknown. Workflow checks use retained operation evidence and +do not depend on the current workflow, so later progress does not invalidate an +authentic replay. Historical repair and stopped-resume events did not independently +seal every presentation field, including the dashboard URL; those fields can be +checked for required shape, but their original bytes cannot always be reconstructed. +The stopped-resume receipt check verifies the form of its admission hash; its owner +separately authenticates the retained admission during execution custody checks. +Pending publication also reports unknown native resource custody when the original +process, root, or lease cannot be proven settled; the existing observer still refuses +admission. After that closure, use the public `repair-admission-preflight --id RUN --request FILE` and `continue-repair --id RUN --request FILE` route with `continuation_kind: "stopped_delivery_resume"`, a stable unique `command_id`, the closed protocol `expected_revision`, `expected_iteration`, and current `expected_candidate_id`/`expected_candidate_head`. A passed implementation stopped with exact unknown prepublication cleanup resumes at the sealed iteration's checks with `additional_iterations: 0`. Its exact durable implementation result and unchanged candidate remain mandatory; all publication, independent review, QA, verification, required CI and tracker gates still run. diff --git a/runtime/src/devflow_temporal/delivery_gate_retry.py b/runtime/src/devflow_temporal/delivery_gate_retry.py index e3496678..10d2260c 100644 --- a/runtime/src/devflow_temporal/delivery_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_gate_retry.py @@ -989,16 +989,18 @@ def admit(store, run_id, payload, *, preflight=False): raise ValueError('gate retry requires an exact zero-repair request') command_digest = digest({'run_id': run_id, **payload}) with store._connect() as db: + db.execute('BEGIN') prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() - if prior: - from .delivery_lifecycle import command_receipt - - assessment = command_receipt(kind, prior, run_id, command_digest) - if preflight: - return {**json.loads(prior['response_json']), 'preflight': True, - 'lifecycle_assessment': assessment} - return json.loads(prior['response_json']) + if prior: + from .delivery_lifecycle import command_receipt + + assessment = command_receipt(kind, prior, run_id, command_digest, + request=payload, db=db) + if preflight: + return {**json.loads(prior['response_json']), 'preflight': True, + 'lifecycle_assessment': assessment} + return json.loads(prior['response_json']) seal = snapshot(store, run_id, kind, _observation=preflight) previous = _retained_recovery(store, run_id, seal) if 'verification_test_paths' in payload: @@ -1044,13 +1046,14 @@ def admit(store, run_id, payload, *, preflight=False): generation, seal['stage'], preflight_generation=preflight_generation) with _lock(root / 'controller.lock'): with store._connect() as db: + db.execute('BEGIN') prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() - if prior: - from .delivery_lifecycle import command_receipt + if prior: + from .delivery_lifecycle import command_receipt - command_receipt(kind, prior, run_id, command_digest) - return json.loads(prior['response_json']) + command_receipt(kind, prior, run_id, command_digest, request=payload, db=db) + return json.loads(prior['response_json']) prepared = None if preflight_generation == 1: from .delivery_gate_preflight_continuation import prepared_handoff diff --git a/runtime/src/devflow_temporal/delivery_lifecycle.py b/runtime/src/devflow_temporal/delivery_lifecycle.py index 214038e8..014c7f36 100644 --- a/runtime/src/devflow_temporal/delivery_lifecycle.py +++ b/runtime/src/devflow_temporal/delivery_lifecycle.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import re import sqlite3 from collections.abc import Mapping from functools import partial @@ -189,9 +190,104 @@ def require_observed_predicates(operation: str, owner: str, proof: Any, raise +def _event_backed_receipt(db, operation, run_id, request, response): + """Bind a gate/publication command to its retained admission and event.""" + from .contracts import canonical_json + + pending = operation == 'recover_publication' + expected_kind = 'pending_publication_retry' if pending else operation + saved = db.execute('SELECT recovery_json FROM delivery_runs WHERE run_id=?', + (run_id,)).fetchone() + roots = [saved[0]] if saved and saved[0] else [] + if not pending: + gate = db.execute('SELECT recovery_json FROM delivery_gate_admissions WHERE run_id=?', + (run_id,)).fetchone() + if gate and gate[0]: + roots.append(gate[0]) + revisions = set() + try: + for raw in roots: + recovery = json.loads(raw) + while isinstance(recovery, dict): + if (recovery.get('kind') == expected_kind + and canonical_json(recovery.get('command')) == canonical_json(request)): + revision = recovery.get('seal', {}).get('row', {}).get('revision') + if type(revision) is int: + revisions.add(revision + 1) + recovery = recovery.get('original_recovery') + if len(revisions) != 1: + return False + event = db.execute( + 'SELECT payload_json FROM delivery_events WHERE run_id=? AND type=? AND ' + 'run_revision=?', + (run_id, 'publication_recovery_queued' if pending else 'gates_retry_queued', + revisions.pop()), + ).fetchall() + return (len(event) == 1 + and canonical_json(json.loads(event[0][0])) == canonical_json(response)) + except (TypeError, ValueError, AttributeError): + return False + + +def _receipt_matches_owner(db, operation, run_id, request_digest, request, response): + """Check historical route evidence without comparing to the current workflow.""" + if not isinstance(response, dict) or response.get('run_id') != run_id: + return False + workflow = response.get('workflow_id') + if not isinstance(workflow, str) or not workflow: + return False + if operation in {'stopped_delivery_resume', 'repair_continuation'}: + if (response.get('phase') != 'repair_continuation_queued' + or not isinstance(response.get('dashboard_url'), str) + or not response['dashboard_url'].endswith('/runs/' + run_id) + or type(response.get('authorized_through_iteration')) is not int): + return False + if operation == 'repair_continuation': + maximum = request['expected_iteration'] + request['additional_iterations'] + grant = db.execute( + 'SELECT granted_iterations,maximum_iteration FROM delivery_repair_grants ' + 'WHERE run_id=? AND command_id=?', (run_id, request['command_id']), + ).fetchone() + return (workflow == f'delivery-{run_id}-repair-continuation-1' + and response.get('existing') is False and grant is not None + and grant['granted_iterations'] == request['additional_iterations'] + and grant['maximum_iteration'] == maximum + and response['authorized_through_iteration'] == maximum) + if (workflow != f'delivery-{run_id}-resume-{request_digest[:20]}' + or not isinstance(response.get('admission_sha256'), str) + or not re.fullmatch(r'[0-9a-f]{64}', response['admission_sha256'])): + return False + try: + events = [json.loads(item[0]) for item in db.execute( + "SELECT payload_json FROM delivery_events WHERE run_id=? " + "AND type='stopped_resume_queued'", (run_id,))] + except (TypeError, ValueError): + return False + admitted = [item for item in events if isinstance(item, dict) + and item.get('command_id') == request['command_id']] + return (len(admitted) == 1 + and type(admitted[0].get('authorized_through_iteration')) is int + and admitted[0]['authorized_through_iteration'] + == response['authorized_through_iteration']) + if operation == 'recover_publication': + valid = (response.get('phase') == 'publication_recovery_queued' + and response.get('implementation_authority') is False) + elif operation in {'published_gate_retry', 'prepublication_gate_retry', + 'published_check_prelaunch_retry', 'published_ci_retry', + 'published_controller_retry'}: + valid = (response.get('phase') == 'gates_retry_queued' + and response.get('implementation_authority') is False + and type(response.get('additional_iterations')) is int + and response['additional_iterations'] == 0 + and response.get('candidate_id') == request['expected_candidate_id']) + else: + return False + return valid and _event_backed_receipt(db, operation, run_id, request, response) + + def command_receipt(operation: str, prior: Mapping[str, Any], run_id: str, - request_digest: str) -> dict[str, Any]: - """Describe one intact prior admission; conflicting or corrupt receipts refuse.""" + request_digest: str, *, request: Mapping[str, Any], db) -> dict[str, Any]: + """Describe a prior admission only when its route-specific receipt is intact.""" from .contracts import digest same = prior['request_digest'] == request_digest @@ -199,14 +295,8 @@ def command_receipt(operation: str, prior: Mapping[str, Any], run_id: str, response = json.loads(prior['response_json']) except (TypeError, ValueError): response = None - expected_phase = { - 'stopped_delivery_resume': 'repair_continuation_queued', - 'repair_continuation': 'repair_continuation_queued', - 'recover_publication': 'publication_recovery_queued', - }.get(operation, 'gates_retry_queued') - intact = (same and prior['run_id'] == run_id and isinstance(response, dict) - and response.get('run_id') == run_id and response.get('phase') == expected_phase - and isinstance(response.get('workflow_id'), str) and bool(response['workflow_id'])) + intact = (same and prior['run_id'] == run_id + and _receipt_matches_owner(db, operation, run_id, request_digest, request, response)) code = ('already_admitted' if intact else 'command_conflict' if not same else 'command_receipt_unknown') fact = ('an existing admission for these exact inputs' if intact else diff --git a/runtime/src/devflow_temporal/delivery_pending_publication.py b/runtime/src/devflow_temporal/delivery_pending_publication.py index ce277d1c..8cbd3799 100644 --- a/runtime/src/devflow_temporal/delivery_pending_publication.py +++ b/runtime/src/devflow_temporal/delivery_pending_publication.py @@ -271,14 +271,16 @@ def admit(store, run_id, payload, *, preflight=False): command_digest = digest({'run_id': run_id, **payload}) with store._connect() as db: + db.execute('BEGIN') prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() - if prior: - assessment = command_receipt('recover_publication', prior, run_id, command_digest) - if preflight: - return {**json.loads(prior['response_json']), 'preflight': True, - 'lifecycle_assessment': assessment} - return json.loads(prior['response_json']) + if prior: + assessment = command_receipt('recover_publication', prior, run_id, command_digest, + request=payload, db=db) + if preflight: + return {**json.loads(prior['response_json']), 'preflight': True, + 'lifecycle_assessment': assessment} + return json.loads(prior['response_json']) seal = snapshot(store, run_id, observation=preflight) previous = _retained_recovery(store, run_id, seal) @@ -300,11 +302,13 @@ def admit(store, run_id, payload, *, preflight=False): root = Path(spec['state_dir']) / namespace with _lock(root / 'controller.lock'): with store._connect() as db: + db.execute('BEGIN') prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() - if prior: - command_receipt('recover_publication', prior, run_id, command_digest) - return json.loads(prior['response_json']) + if prior: + command_receipt('recover_publication', prior, run_id, command_digest, + request=payload, db=db) + return json.loads(prior['response_json']) execution = prepare_runtime(spec, root, command_digest, digest(seal)) require_same_proof('recover_publication', 'pending_publication', digest(snapshot(store, run_id)), digest(seal)) diff --git a/runtime/src/devflow_temporal/delivery_stopped_resume.py b/runtime/src/devflow_temporal/delivery_stopped_resume.py index 5c38e91f..208480b6 100644 --- a/runtime/src/devflow_temporal/delivery_stopped_resume.py +++ b/runtime/src/devflow_temporal/delivery_stopped_resume.py @@ -1339,16 +1339,18 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): raise ValueError('stopped resume requires a finite exact-checkpoint command') command_digest = digest({'run_id': run_id, **command}) with store._connect() as db: + db.execute('BEGIN') prior = db.execute('SELECT run_id,request_digest,response_json FROM delivery_commands ' 'WHERE command_id=?', (command['command_id'],)).fetchone() - if prior: - from .delivery_lifecycle import command_receipt - - assessment = command_receipt(KIND, prior, run_id, command_digest) - if preflight: - return {**json.loads(prior['response_json']), 'preflight': True, - 'lifecycle_assessment': assessment} - return json.loads(prior['response_json']) + if prior: + from .delivery_lifecycle import command_receipt + + assessment = command_receipt(KIND, prior, run_id, command_digest, + request=command, db=db) + if preflight: + return {**json.loads(prior['response_json']), 'preflight': True, + 'lifecycle_assessment': assessment} + return json.loads(prior['response_json']) seal = snapshot(store, run_id, _revision=_revision, _observation=preflight) gate_only = None if _revision is not None: @@ -1468,15 +1470,16 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): root = Path(spec['state_dir']) / 'stopped-resumes' / command_digest with _lock(Path(spec['state_dir']) / 'stopped-resumes' / 'controller.lock'): with store._connect() as db: + db.execute('BEGIN') prior = db.execute( 'SELECT run_id,request_digest,response_json FROM delivery_commands ' 'WHERE command_id=?', (command['command_id'],)).fetchone() - if prior: - from .delivery_lifecycle import command_receipt + if prior: + from .delivery_lifecycle import command_receipt - command_receipt(KIND, prior, run_id, command_digest) - return json.loads(prior['response_json']) + command_receipt(KIND, prior, run_id, command_digest, request=command, db=db) + return json.loads(prior['response_json']) # Recheck all stopped readbacks after lock acquisition, before any native work. require_same_proof(KIND, 'delivery_stopped_resume', digest(snapshot(store, run_id, _revision=_revision)), digest(seal)) diff --git a/runtime/src/devflow_temporal/delivery_store.py b/runtime/src/devflow_temporal/delivery_store.py index 0467b290..712bd967 100644 --- a/runtime/src/devflow_temporal/delivery_store.py +++ b/runtime/src/devflow_temporal/delivery_store.py @@ -1987,12 +1987,14 @@ def continue_repair( raise ValueError("invalid repair continuation identity or grant") command_digest = digest({"run_id": run_id, **supplied}) with self._connect() as db: + db.execute("BEGIN") prior = db.execute( "SELECT run_id,request_digest,response_json FROM delivery_commands " "WHERE command_id=?", (command_id,), ).fetchone() if prior: - receipt = command_receipt('repair_continuation', prior, run_id, command_digest) + receipt = command_receipt('repair_continuation', prior, run_id, command_digest, + request=supplied, db=db) if preflight: return {**json.loads(prior["response_json"]), "preflight": True, "lifecycle_assessment": receipt} @@ -2291,7 +2293,8 @@ def observe_admission(db): "WHERE command_id=?", (command_id,), ).fetchone() if prior: - command_receipt('repair_continuation', prior, run_id, command_digest) + command_receipt('repair_continuation', prior, run_id, command_digest, + request=supplied, db=db) return json.loads(prior["response_json"]) current = observe_admission(db) if stopped_claim: diff --git a/runtime/tests/test_delivery_feature_resumed_evidence.py b/runtime/tests/test_delivery_feature_resumed_evidence.py index 373b6e9b..ce83692d 100644 --- a/runtime/tests/test_delivery_feature_resumed_evidence.py +++ b/runtime/tests/test_delivery_feature_resumed_evidence.py @@ -235,10 +235,17 @@ def observed_admission(owner, run_id, command, **kwargs): with store._connect() as db: older = json.loads(db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id=?", (child["run_id"],)).fetchone()[0]) + first_receipt = json.loads(db.execute( + "SELECT response_json FROM delivery_commands WHERE command_id=?", + (older["command"]["command_id"],)).fetchone()[0]) # The resumed execution is cancelled again; a second admission chains onto the first. cancelled(store, monkeypatch, parent, child, role, output, workflow_id=first["workflow_id"], execution="closed-resume", recovery_digest=digest(older), revision=9, finish=False) second = activities.resume_worker(store, parent, child, row) + # The first command still identifies its original admission after the later resume. + preview = original_admit(store, child["run_id"], older["command"], preflight=True) + assert preview["lifecycle_assessment"]["disposition"] == "already_admitted" + assert original_admit(store, child["run_id"], older["command"]) == first_receipt specs = {"current": second["spec"], "middle": first["spec"], "oldest": child} assert len({digest(value) for value in specs.values()}) == 3 if ran_under != "oldest": diff --git a/runtime/tests/test_delivery_gate_retry.py b/runtime/tests/test_delivery_gate_retry.py index 900a8412..682e0a15 100644 --- a/runtime/tests/test_delivery_gate_retry.py +++ b/runtime/tests/test_delivery_gate_retry.py @@ -180,7 +180,7 @@ def first_published_after_repair( (canonical_json(state['checks']),)) grant = {k: v for k, v in request.items() if k != 'continuation_kind'} grant.update(command_id='one-source-grant', additional_iterations=1) - store.continue_repair('run-1', grant) + original_receipt = store.continue_repair('run-1', grant) with store._connect() as db: prior = json.loads(db.execute( "SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0]) @@ -198,6 +198,8 @@ def first_published_after_repair( request.update(command_id='post-repair-assessment', expected_iteration=2, expected_revision=14) before_grant = canonical_json(prior) store.continue_repair('run-1', request) + # An earlier authentic command remains replayable after the workflow advances. + assert store.continue_repair('run-1', grant) == original_receipt effective = store.effective_spec('run-1') assert effective['gate_retry_stage'] == 'published' assert effective['policy']['max_repairs'] == store.spec('run-1')['policy']['max_repairs'] @@ -211,6 +213,44 @@ def first_published_after_repair( return store, broker, state, closed, request, current +@pytest.mark.asyncio +async def test_public_historical_repair_replay_after_gate_admission(stopped, monkeypatch): + """A later gate workflow must not invalidate the original repair receipt.""" + import httpx + + from devflow_temporal.delivery_api import create_app + + store, _, _, _, _, current = first_published_after_repair(stopped, monkeypatch) + original = current['original_recovery'] + state = original['state'] + request = {'command_id': 'one-source-grant', 'expected_revision': state['revision'], + 'expected_iteration': state['iteration'], + 'expected_candidate_id': original['candidate']['id'], + 'expected_pr_number': state['pull_request']['number'], + 'expected_pr_head': state['pull_request']['head'], + 'additional_iterations': original['additional_iterations']} + with store._connect() as db: + expected = json.loads(db.execute( + 'SELECT response_json FROM delivery_commands WHERE command_id=?', + (request['command_id'],)).fetchone()[0]) + before = tuple(db.execute(f'SELECT count(*) FROM {table}').fetchone()[0] + for table in ('delivery_commands', 'delivery_repair_grants')) + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': 'repair_continuation', 'request': request}) + replay = await client.post('/api/runs/run-1/continue-repair', json=request, + headers={'Origin': host, 'X-Devflow-CSRF': token}) + assert preview.json()['disposition'] == 'already_admitted' + assert replay.status_code == 200 and replay.json() == expected + with store._connect() as db: + assert tuple(db.execute(f'SELECT count(*) FROM {table}').fetchone()[0] + for table in ('delivery_commands', 'delivery_repair_grants')) == before + + @pytest.fixture def missing_report(stopped, monkeypatch): from copy import deepcopy @@ -251,7 +291,10 @@ def missing_report(stopped, monkeypatch): @pytest.mark.asyncio async def test_public_gate_retry_rechecks_candidate_and_owner(unpublished, monkeypatch): """A valid preview cannot authorize changed source or a competing owner.""" + import asyncio import copy + import threading + from contextlib import contextmanager import httpx @@ -316,9 +359,24 @@ async def assess(): assert not db.execute('SELECT 1 FROM delivery_gate_admissions').fetchone() store.state.release_work(db, broker.spec['work_id'], 'external:other') assert (await assess()).json()['eligible'] - accepted = await client.post('/api/runs/run-1/continue-repair', json=command, - headers=headers) - assert accepted.status_code == 200, accepted.text + # Force both requests to observe the missing command before owner locking. + original_lock = delivery_gate_retry._lock + ready = threading.Barrier(2, timeout=10) + + @contextmanager + def overlapping_lock(path): + ready.wait() + with original_lock(path): + yield + + monkeypatch.setattr(delivery_gate_retry, '_lock', overlapping_lock) + accepted, concurrent = await asyncio.gather(*( + client.post('/api/runs/run-1/continue-repair', json=command, headers=headers) + for _ in range(2))) + assert accepted.status_code == concurrent.status_code == 200, ( + accepted.text, concurrent.text) + assert concurrent.json() == accepted.json() + monkeypatch.setattr(delivery_gate_retry, '_lock', original_lock) replay = await client.post('/api/runs/run-1/continue-repair', json=command, headers=headers) assert replay.json() == accepted.json() @@ -340,13 +398,18 @@ async def assess(): with store._connect() as db: saved = db.execute('SELECT response_json FROM delivery_commands WHERE command_id=?', (command['command_id'],)).fetchone()[0] - db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', - ('{}', command['command_id'])) - corrupt = await assess() - refused = await client.post('/api/runs/run-1/continue-repair', - json=command, headers=headers) - assert 'command_receipt_unknown' in {b['code'] for b in corrupt.json()['blockers']} - assert refused.status_code == 409 and refused.json()['detail'] == corrupt.json() + receipt = accepted.json() + for damaged in ({}, {**receipt, 'workflow_id': 'unrelated-workflow'}, + {key: receipt[key] for key in ('run_id', 'phase', 'workflow_id')}, + {**receipt, 'additional_iterations': 1}): + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (json.dumps(damaged), command['command_id'])) + corrupt = await assess() + refused = await client.post('/api/runs/run-1/continue-repair', + json=command, headers=headers) + assert 'command_receipt_unknown' in {b['code'] for b in corrupt.json()['blockers']} + assert refused.status_code == 409 and refused.json()['detail'] == corrupt.json() with store._connect() as db: db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', (saved, command['command_id'])) diff --git a/runtime/tests/test_delivery_pending_publication.py b/runtime/tests/test_delivery_pending_publication.py index acae17af..c89a75ae 100644 --- a/runtime/tests/test_delivery_pending_publication.py +++ b/runtime/tests/test_delivery_pending_publication.py @@ -122,6 +122,16 @@ async def test_public_pending_assessment_agrees_with_admission(pending): assert accepted.status_code == 200 and not accepted.json()['implementation_authority'] replay = await client.post(path + 'recover-publication', json=request, headers=headers) assert replay.status_code == 200 and replay.json() == accepted.json() + # The prior gate retry remains readable after publication starts a new workflow. + with store._connect() as db: + prior_gate = json.loads(db.execute( + 'SELECT recovery_json FROM delivery_gate_admissions WHERE run_id=?', + ('run-1',)).fetchone()[0])['command'] + prior_gate_response = json.loads(db.execute( + 'SELECT response_json FROM delivery_commands WHERE command_id=?', + (prior_gate['command_id'],)).fetchone()[0]) + historical = await client.post(path + 'continue-repair', json=prior_gate, headers=headers) + assert historical.status_code == 200 and historical.json() == prior_gate_response replay_preview = await client.post(path + 'assess-transition', json={ 'kind': 'recover_publication', 'request': request}) conflict = {**request, 'expected_revision': request['expected_revision'] + 1} @@ -138,12 +148,25 @@ async def test_public_pending_assessment_agrees_with_admission(pending): with store._connect() as db: assert db.execute('SELECT count(*) FROM delivery_commands WHERE command_id=?', (request['command_id'],)).fetchone()[0] == 1 + saved = db.execute('SELECT response_json FROM delivery_commands WHERE command_id=?', + (request['command_id'],)).fetchone()[0] + receipt = accepted.json() + for corrupt in ({}, {**receipt, 'workflow_id': 'unrelated-workflow'}, + {key: receipt[key] for key in ('run_id', 'phase', 'workflow_id')}, + {**receipt, 'implementation_authority': True}): + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (json.dumps(corrupt), request['command_id'])) + corrupt_preview = await client.post(path + 'assess-transition', json={ + 'kind': 'recover_publication', 'request': request}) + corrupt_admission = await client.post(path + 'recover-publication', json=request, + headers=headers) + assert corrupt_admission.status_code == 409 + assert corrupt_admission.json()['detail'] == corrupt_preview.json() + assert 'command_receipt_unknown' in { + b['code'] for b in corrupt_preview.json()['blockers']} + with store._connect() as db: db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', - ('{}', request['command_id'])) - corrupt_preview = await client.post(path + 'assess-transition', json={ - 'kind': 'recover_publication', 'request': request}) - corrupt_admission = await client.post(path + 'recover-publication', json=request, - headers=headers) - assert corrupt_admission.status_code == 409 - assert corrupt_admission.json()['detail'] == corrupt_preview.json() - assert 'command_receipt_unknown' in {b['code'] for b in corrupt_preview.json()['blockers']} + (saved, request['command_id'])) + assert (await client.post(path + 'recover-publication', json=request, + headers=headers)).json() == receipt diff --git a/runtime/tests/test_delivery_store_lifecycle_240.py b/runtime/tests/test_delivery_store_lifecycle_240.py index 30e20c49..0977263d 100644 --- a/runtime/tests/test_delivery_store_lifecycle_240.py +++ b/runtime/tests/test_delivery_store_lifecycle_240.py @@ -31,6 +31,8 @@ def test_scope_stale_request_never_mutates(service, monkeypatch, field, value): @pytest.mark.asyncio async def test_default_repair_public_assessment_and_admission_agree(service, monkeypatch): + import json + import httpx from test_delivery_store import _failed_published_repair_fixture @@ -164,15 +166,33 @@ async def test_default_repair_public_assessment_and_admission_agree(service, mon (command['command_id'],)).fetchone()[0] == 1 assert not db.execute('SELECT 1 FROM delivery_commands WHERE command_id=?', (another['command_id'],)).fetchone() + saved = db.execute('SELECT response_json FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone()[0] + receipt = accepted.json() + corruptions = [ + {}, + {**receipt, 'workflow_id': 'unrelated-workflow'}, + {key: receipt[key] for key in ('run_id', 'phase', 'workflow_id')}, + {**receipt, 'authorized_through_iteration': + receipt['authorized_through_iteration'] + 1}, + ] + for corrupt in corruptions: + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (json.dumps(corrupt), command['command_id'])) + corrupt_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + corrupt_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert corrupt_admission.status_code == 409 + assert corrupt_admission.json()['detail'] == corrupt_preview.json() + assert 'command_receipt_unknown' in { + b['code'] for b in corrupt_preview.json()['blockers']} + with store._connect() as db: db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', - ('{}', command['command_id'])) - corrupt_preview = await client.post(base + 'assess-transition', json={ - 'kind': 'repair_continuation', 'request': command}) - corrupt_admission = await client.post(base + 'continue-repair', json=command, - headers=headers) - assert corrupt_admission.status_code == 409 - assert corrupt_admission.json()['detail'] == corrupt_preview.json() - assert 'command_receipt_unknown' in {b['code'] for b in corrupt_preview.json()['blockers']} + (saved, command['command_id'])) + assert (await client.post(base + 'continue-repair', json=command, + headers=headers)).json() == receipt @pytest.mark.asyncio diff --git a/runtime/tests/test_delivery_unpublished_cleanup.py b/runtime/tests/test_delivery_unpublished_cleanup.py index ff181cd0..12c2ff34 100644 --- a/runtime/tests/test_delivery_unpublished_cleanup.py +++ b/runtime/tests/test_delivery_unpublished_cleanup.py @@ -225,14 +225,21 @@ async def test_public_unpublished_cleanup_then_existing_finite_continuation(clos with store._connect() as db: saved = db.execute('SELECT response_json FROM delivery_commands WHERE command_id=?', (command['command_id'],)).fetchone()[0] - db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', - ('{}', command['command_id'])) - corrupt = await client.post('/api/runs/run-1/assess-transition', json={ - 'kind': resume.KIND, 'request': command}) - rejected = await client.post('/api/runs/run-1/continue-repair', json=command, - headers=headers) - assert 'command_receipt_unknown' in {b['code'] for b in corrupt.json()['blockers']} - assert rejected.status_code == 409 and rejected.json()['detail'] == corrupt.json() + receipt = response.json() + for damaged in ({}, {**receipt, 'workflow_id': 'unrelated-workflow'}, + {key: receipt[key] for key in ('run_id', 'phase', 'workflow_id')}, + {**receipt, 'authorized_through_iteration': + receipt['authorized_through_iteration'] + 1}, + {**receipt, 'admission_sha256': 'invalid'}): + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (json.dumps(damaged), command['command_id'])) + corrupt = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + rejected = await client.post('/api/runs/run-1/continue-repair', json=command, + headers=headers) + assert 'command_receipt_unknown' in {b['code'] for b in corrupt.json()['blockers']} + assert rejected.status_code == 409 and rejected.json()['detail'] == corrupt.json() with store._connect() as db: db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', (saved, command['command_id'])) From e5671ea1fef3ad5ae6c4bb5a97ff1c3f04f7582b Mon Sep 17 00:00:00 2001 From: Eloi Date: Sun, 1 Nov 2026 18:45:35 +0100 Subject: [PATCH 21/24] fix(recovery): bind retained state reads to opened files --- .../delivery_retained_checkpoint.py | 89 ++++++++++++++++--- 1 file changed, 77 insertions(+), 12 deletions(-) diff --git a/runtime/src/devflow_temporal/delivery_retained_checkpoint.py b/runtime/src/devflow_temporal/delivery_retained_checkpoint.py index 4757edde..c7cae1dc 100644 --- a/runtime/src/devflow_temporal/delivery_retained_checkpoint.py +++ b/runtime/src/devflow_temporal/delivery_retained_checkpoint.py @@ -287,22 +287,87 @@ def _verify_recorded_source(spec, value, checkpoint_root, resources): "branch": branch, "head": head, "index_tree": value["source"]["index_tree"]} +def _retained_state_bytes(target, expected, size): + """Return validated bytes, or a coherently observed initial absence.""" + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_NONBLOCK + with ExitStack() as stack: + parent = os.open(target.anchor, flags) + stack.callback(os.close, parent) + directories = [] + + def parents_unchanged(): + keys = ("st_dev", "st_ino", "st_mode", "st_uid") + try: + for owner, name, descriptor, initial in directories: + current = os.fstat(descriptor) + named = os.stat(name, dir_fd=owner, follow_symlinks=False) + if (any(getattr(initial, key) != getattr(current, key) for key in keys) + or any(getattr(current, key) != getattr(named, key) for key in keys)): + raise ValueError("retained checkpoint state parent changed") + except FileNotFoundError as exc: + raise ValueError("retained checkpoint state parent became unavailable") from exc + + def open_entry(name, entry_flags): + try: + return os.open(name, entry_flags, dir_fd=parent) + except FileNotFoundError: + parents_unchanged() + try: + os.stat(name, dir_fd=parent, follow_symlinks=False) + except FileNotFoundError: + parents_unchanged() + return None + raise ValueError("retained checkpoint state entry changed") from None + + for name in target.parts[1:-1]: + opened = open_entry(name, flags) + if opened is None: + return None + stack.callback(os.close, opened) + directories.append((parent, name, opened, os.fstat(opened))) + parent = opened + if expected is not None and os.fstat(parent).st_uid != os.getuid(): + raise ValueError("retained checkpoint destination parent is not owned") + opened = open_entry(target.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + if opened is None: + return None + stack.callback(os.close, opened) + before = os.fstat(opened) + if expected is None: + raise ValueError("retained checkpoint no-GO absence changed") + if (not stat.S_ISREG(before.st_mode) or before.st_uid != os.getuid() + or before.st_nlink != 1 or before.st_size != size + or stat.S_IMODE(before.st_mode) != expected["mode"] + or before.st_mtime_ns != expected["mtime_ns"]): + raise ValueError("retained checkpoint evidence metadata changed") + with os.fdopen(os.dup(opened), "rb") as stream: + raw = stream.read(size + 1) + keys = ("st_dev", "st_ino", "st_mode", "st_uid", "st_nlink", "st_size", + "st_mtime_ns", "st_ctime_ns") + after = os.fstat(opened) + try: + named = os.stat(target.name, dir_fd=parent, follow_symlinks=False) + except FileNotFoundError as exc: + raise ValueError("retained checkpoint evidence became unavailable") from exc + if (len(raw) != size + or any(getattr(before, key) != getattr(after, key) for key in keys) + or any(getattr(after, key) != getattr(named, key) for key in keys)): + raise ValueError("retained checkpoint evidence changed during observation") + parents_unchanged() + return raw + + def verify_retained_state(spec, value, checkpoint_root, *, allow_missing=False): """Verify state targets; missing files can be restored when explicitly allowed.""" for relative, item in value["state_refs"].items(): target = _path(Path(spec["state_dir"]), relative) - try: - info = target.lstat() - except FileNotFoundError: + expected = (None if item.get("kind") == "absent" else + read_private(checkpoint_root / item["artifact"], raw=True)) + raw = _retained_state_bytes(target, item if expected is not None else None, + len(expected) if expected is not None else 0) + if raw is None: if item.get("kind") == "absent" or allow_missing: continue - raise - if item.get("kind") == "absent": - raise ValueError("retained checkpoint no-GO absence changed: " + relative) - if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1: - raise ValueError("retained checkpoint evidence is not an owned file: " + relative) - raw = target.read_bytes() - if (raw != read_private(checkpoint_root / item["artifact"], raw=True) - or _hash(raw) != item["sha256"] or stat.S_IMODE(info.st_mode) != item["mode"] - or info.st_mtime_ns != item["mtime_ns"]): + raise FileNotFoundError("retained checkpoint evidence is absent: " + relative) + if raw != expected or _hash(raw) != item["sha256"]: raise ValueError("retained checkpoint original evidence changed: " + relative) From 479c56c245120f764f1294dffaecf82f0f0ed7f0 Mon Sep 17 00:00:00 2001 From: Eloi Date: Sun, 1 Nov 2026 19:00:09 +0100 Subject: [PATCH 22/24] feat(runtime): guard staged publication handoffs --- .../devflow_temporal/delivery_activities.py | 30 +- .../src/devflow_temporal/delivery_broker.py | 86 +++- .../src/devflow_temporal/delivery_pause.py | 117 +++++ .../delivery_pause_context.py | 81 +++- .../delivery_publication_execution.py | 29 +- .../test_delivery_publication_execution.py | 405 ++++++++++++++++++ 6 files changed, 729 insertions(+), 19 deletions(-) diff --git a/runtime/src/devflow_temporal/delivery_activities.py b/runtime/src/devflow_temporal/delivery_activities.py index bca456e6..fa6816e5 100644 --- a/runtime/src/devflow_temporal/delivery_activities.py +++ b/runtime/src/devflow_temporal/delivery_activities.py @@ -14,6 +14,7 @@ import sys import threading import time +from dataclasses import replace from pathlib import Path from typing import Any @@ -65,12 +66,20 @@ def database_contention(exc: BaseException) -> bool: class _DatabaseContentionInbound(ActivityInboundInterceptor): async def execute_activity(self, input: ExecuteActivityInput) -> Any: + from .delivery_pause import PauseDeferred + from .delivery_pause_context import activate, split_activity_request + try: # Only fresh ordinary histories opt in; legacy and merge workflows # retain their existing protocols. SDK context selects the execution. from . import delivery_execution_authority as authority info = activity.info() - request = input.args[0] if input.args else None + args = list(input.args) + context = None + if len(args) == 1: + args[0], context = split_activity_request(args[0], info) + adjusted = replace(input, args=args) if context is not None else input + request = args[0] if args else None spec = request.get("spec") if isinstance(request, dict) else None if (info.workflow_type == "DevflowDeliveryWorkflow" and isinstance(spec, dict) and spec.get(authority.VERSION) == 1): @@ -81,10 +90,21 @@ async def execute_activity(self, input: ExecuteActivityInput) -> Any: from .delivery_writer_custody import activity_custody async with activity_custody(spec): - return await super().execute_activity(input) + with activate(context): + return await super().execute_activity(adjusted) finally: authority._ACTIVITY_AUTHORITY.reset(token) - return await super().execute_activity(input) + with activate(context): + return await super().execute_activity(adjusted) + except PauseDeferred as exc: + if (not exc.root_id or not exc.operation_key or not exc.step + or type(exc.epoch) is not int): + raise + raise ApplicationError( + str(exc), {"root_id": exc.root_id, "operation_key": exc.operation_key, + "step": exc.step, "epoch": exc.epoch}, + type="PauseDeferred", non_retryable=True, + ) from exc except sqlite3.OperationalError as exc: if not database_contention(exc): raise @@ -1099,6 +1119,7 @@ def _role_result(request, broker, workspace, review_diff, result): @activity.defn(name="delivery_publish") async def delivery_publish(request: dict[str, Any]) -> dict[str, Any]: + from .delivery_pause import PauseDeferred from .delivery_writer_custody import to_thread def execute() -> dict[str, Any]: @@ -1109,6 +1130,9 @@ def execute() -> dict[str, Any]: return publish(store, broker, request) return broker.publish(request["iteration"], request["candidate"]) + except PauseDeferred: + # The owner proved that this exact handoff never launched. + raise except Exception as exc: if (request["spec"].get("publication_readback_version") == 1 and broker.publication_may_have_effect is False): diff --git a/runtime/src/devflow_temporal/delivery_broker.py b/runtime/src/devflow_temporal/delivery_broker.py index 090c2d62..6f433502 100644 --- a/runtime/src/devflow_temporal/delivery_broker.py +++ b/runtime/src/devflow_temporal/delivery_broker.py @@ -28,25 +28,67 @@ def _run(argv: list[str], *, cwd: Path | None = None, timeout: int = 120) -> str: + from .delivery_pause_context import enabled as pause_enabled + from .delivery_publication_execution import current_mutation from .delivery_publication_execution import inherited_descriptors as publication_descriptors from .delivery_writer_custody import inherited_descriptors as writer_descriptors nul_output = argv[0] == 'git' and '-z' in argv - result = subprocess.run( - argv, cwd=cwd, text=not nul_output, capture_output=True, timeout=timeout, check=False, - pass_fds=tuple(dict.fromkeys((*publication_descriptors(), *writer_descriptors()))), - ) - stdout = result.stdout.decode('utf-8', 'surrogateescape') if nul_output else result.stdout - stderr = result.stderr.decode('utf-8', 'surrogateescape') if nul_output else result.stderr - if result.returncode: + pass_fds = tuple(dict.fromkeys((*publication_descriptors(), *writer_descriptors()))) + mutation = current_mutation() + if mutation is None or not pause_enabled(mutation[0].broker.spec): + result = subprocess.run( + argv, cwd=cwd, text=not nul_output, capture_output=True, timeout=timeout, check=False, + pass_fds=pass_fds, + ) + returncode, stdout, stderr = result.returncode, result.stdout, result.stderr + else: + from .delivery_pause_context import publication_handoff + + owner, step = mutation + with publication_handoff(owner, step) as handoff: + try: + process = subprocess.Popen(argv, cwd=cwd, text=not nul_output, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + pass_fds=pass_fds) + except (FileNotFoundError, PermissionError): + handoff.no_process_started() + raise + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired as exc: + process.kill() + stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired(argv, timeout, output=stdout, stderr=stderr) from exc + except BaseException: + process.kill() + process.communicate() + raise + returncode = process.returncode + stdout = stdout.decode('utf-8', 'surrogateescape') if nul_output else stdout + stderr = stderr.decode('utf-8', 'surrogateescape') if nul_output else stderr + if returncode: raise RuntimeError( - f"command failed ({result.returncode}): {argv[0]} {argv[1]}: " + f"command failed ({returncode}): {argv[0]} {argv[1]}: " + (stderr.strip() or stdout.strip())[:500] ) # Preserve exact filenames, including CR/LF and leading/trailing whitespace. return stdout if nul_output else stdout.strip() +def _staged_index_evidence(checkout: Path) -> dict[str, str]: + """Observe supported index semantics without mutating or restaging.""" + from .delivery_checkpoint_index import observe_index + + def query(path, *args): + return _git(path, *args).encode("utf-8", "surrogateescape") + + try: + return observe_index(checkout, query) + except ValueError as exc: + raise ValueError("staged publication index semantics are unproven") from exc + + def _git(path: Path, *args: str) -> str: return _run( [ @@ -1708,6 +1750,12 @@ def publish(self, iteration: int, input_candidate: dict[str, Any]) -> dict[str, return {"state": "pending", "reason": "original_" + exc.operation + "_readback"} def _publish_owned(self, iteration, input_candidate, owner): + from .delivery_pause_context import enabled as pause_enabled + + pause_stage = pause_enabled(self.spec) + if pause_stage: + # Unsupported index flags can cause Git to omit reviewed worktree bytes. + _staged_index_evidence(self.checkout) # Only this invocation's confirmed pre-mutation failure can release uncertainty. # Any retained original effect may belong to an earlier lost completion. if self.spec.get("publication_readback_version") == 1: @@ -1767,7 +1815,27 @@ def _publish_owned(self, iteration, input_candidate, owner): if path in indexed or os.path.lexists(self.checkout / path)) # Already-staged removals are absent from both index and worktree; # preserve them without passing an unmatched pathspec to git add. - if stageable: + stage = owner.mutation("stage") if pause_stage else {} + if stage.get("state") == "complete": + # Path-only diffs ignore untracked files and index flags. Bind every + # logical index entry recorded after staging, even for pure deletions. + if stage.get("evidence") != _staged_index_evidence(self.checkout): + raise ValueError("staged publication candidate changed during pause") + unstaged = set(_git(self.checkout, "diff", "--name-only", "-z").split("\0")) + staged = set(_git( + self.checkout, "diff", "--cached", "--name-only", "-z").split("\0")) + unstaged.discard("") + staged.discard("") + if unstaged or staged != set(changed): + raise ValueError("staged publication candidate changed during pause") + elif pause_stage: + # Even an already-staged deletion needs a recorded index before + # the commit fence, and an uncertain earlier stage cannot be replayed. + owner.mutate("stage", lambda: _git( + self.checkout, *(['--literal-pathspecs', 'add', '--all', '--', *stageable] + if stageable else ['ls-files', '--stage', '-z'])), + observe_completion=lambda: _staged_index_evidence(self.checkout)) + elif stageable: _git(self.checkout, '--literal-pathspecs', 'add', '--all', '--', *stageable) if _git(self.checkout, "diff", "--cached", "--name-only"): owner.mutate("commit", lambda: _git( diff --git a/runtime/src/devflow_temporal/delivery_pause.py b/runtime/src/devflow_temporal/delivery_pause.py index 03c99bb1..95504816 100644 --- a/runtime/src/devflow_temporal/delivery_pause.py +++ b/runtime/src/devflow_temporal/delivery_pause.py @@ -48,6 +48,11 @@ def initialize(db): epoch INTEGER NOT NULL, workflow_id TEXT NOT NULL, execution_run_id TEXT NOT NULL, owner_generation INTEGER NOT NULL, admitted_at TEXT NOT NULL, PRIMARY KEY(root_id, operation_key, step))""") + db.execute("""CREATE TABLE IF NOT EXISTS delivery_pause_denials ( + root_id TEXT NOT NULL, operation_key TEXT NOT NULL, step TEXT NOT NULL, + epoch INTEGER NOT NULL, workflow_id TEXT NOT NULL, execution_run_id TEXT NOT NULL, + owner_generation INTEGER NOT NULL, denied_at TEXT NOT NULL, + PRIMARY KEY(root_id, operation_key, step, epoch))""") db.execute("""CREATE TABLE IF NOT EXISTS delivery_pause_pending_merges ( root_id TEXT NOT NULL, workflow_id TEXT NOT NULL, grant_sha256 TEXT NOT NULL, state TEXT NOT NULL, execution_run_id TEXT, PRIMARY KEY(root_id, workflow_id))""") @@ -241,6 +246,118 @@ def _missing_clock(db, root_id, epoch, *, finished): + condition + ") LIMIT 1", (epoch, root_id)).fetchone() is not None +@contextmanager +def launch_admission_graph(store, bindings, operation_key, step, *, workflow_id, + execution_run_id, owner_generation, validate_graph, graph_guard=None): + """Admit one actual handoff under every authenticated ancestor fence. + + The caller owns actor and graph authentication; this internal interface does + not accept a validator from a public request. Locks are retained through the + caller's handoff, but the tracking write transaction is committed first. + An admitted step cannot be repeated without a separate readback proving the + outcome. A denial is recorded before it is returned to the owning workflow. + """ + from contextlib import ExitStack + + if not isinstance(bindings, dict) or not bindings or not callable(validate_graph): + raise ValueError("pause graph bindings and validator are required") + for value, name in ((operation_key, "operation key"), (step, "step"), + (workflow_id, "workflow ID"), (execution_run_id, "execution run ID")): + _nonempty(value, name) + if type(owner_generation) is not int or owner_generation < 1: + raise ValueError("invalid pause actor generation") + for root_id, epoch in bindings.items(): + _nonempty(root_id, "root ID") + _epoch(epoch) + + class Handoff: + active = True + + def no_process_started(self): + """Withdraw only after the immediate Popen reports a pre-exec failure.""" + if not self.active: + raise ValueError("pause handoff no longer holds its admission locks") + with store._connect() as db: + db.execute("BEGIN IMMEDIATE") + validate_graph(db, bindings, workflow_id, execution_run_id, owner_generation) + for root_id, epoch in sorted(bindings.items()): + row = _row(db, root_id) + actor = db.execute("""SELECT a.execution_run_id,a.owner_generation,s.state + FROM delivery_pause_actors a JOIN delivery_pause_actor_status s + ON s.root_id=a.root_id AND s.workflow_id=a.workflow_id + WHERE a.root_id=? AND a.workflow_id=?""", + (root_id, workflow_id)).fetchone() + if (row["state"] != "running" or row["epoch"] != epoch or actor is None + or actor["state"] != "running" + or actor["execution_run_id"] != execution_run_id + or actor["owner_generation"] != owner_generation): + raise ValueError("pause actor changed before no-launch acknowledgement") + removed = db.execute("""DELETE FROM delivery_pause_steps WHERE root_id=? + AND operation_key=? AND step=? AND epoch=? AND workflow_id=? + AND execution_run_id=? AND owner_generation=?""", + (root_id, operation_key, step, epoch, workflow_id, + execution_run_id, owner_generation)).rowcount + if removed != 1: + raise ValueError("pause handoff no-launch identity changed") + self.active = False + + with ExitStack() as stack: + for root_id in sorted(bindings): + stack.enter_context(root_mutex(store, root_id)) + if graph_guard is not None: + stack.enter_context(graph_guard()) + denied = None + with store._connect() as db: + db.execute("BEGIN IMMEDIATE") + validate_graph(db, bindings, workflow_id, execution_run_id, owner_generation) + controls = [] + for root_id in sorted(bindings): + row = _row(db, root_id) + actor = db.execute("""SELECT a.execution_run_id,a.owner_generation,s.state + FROM delivery_pause_actors a JOIN delivery_pause_actor_status s + ON s.root_id=a.root_id AND s.workflow_id=a.workflow_id + WHERE a.root_id=? AND a.workflow_id=?""", (root_id, workflow_id)).fetchone() + if (actor is None or actor["execution_run_id"] != execution_run_id + or actor["owner_generation"] != owner_generation + or actor["state"] not in {"running", "resume_pending"}): + raise ValueError("pause graph actor changed or completed") + controls.append((row, actor["state"])) + for row, actor_state in controls: + if (row["state"] == "running" and actor_state == "running" + and row["epoch"] == bindings[row["root_id"]]): + continue + db.execute("""INSERT OR IGNORE INTO delivery_pause_denials + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", (row["root_id"], operation_key, step, + row["epoch"], workflow_id, execution_run_id, owner_generation, _now())) + existing = db.execute("""SELECT * FROM delivery_pause_denials WHERE root_id=? + AND operation_key=? AND step=? AND epoch=?""", + (row["root_id"], operation_key, step, row["epoch"])).fetchone() + if (existing["workflow_id"] != workflow_id + or existing["execution_run_id"] != execution_run_id + or existing["owner_generation"] != owner_generation): + raise ValueError("pause denial belongs to a different actor") + if denied is None: + denied = row + if denied is None: + for row, _actor_state in controls: + old = db.execute("""SELECT 1 FROM delivery_pause_steps WHERE root_id=? + AND operation_key=? AND step=?""", + (row["root_id"], operation_key, step)).fetchone() + if old is not None: + raise ValueError("pause handoff already admitted or outcome unknown") + db.execute("INSERT INTO delivery_pause_steps VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (row["root_id"], operation_key, step, row["epoch"], workflow_id, + execution_run_id, owner_generation, _now())) + if denied is not None: + raise PauseDeferred("pause fence closed before handoff", root_id=denied["root_id"], + operation_key=operation_key, step=step, epoch=denied["epoch"]) + handoff = Handoff() + try: + yield handoff + finally: + handoff.active = False + + @contextmanager def start_admission(store, root_id, workflow_id, owner_generation, *, epoch, validate, retry_selected=False): diff --git a/runtime/src/devflow_temporal/delivery_pause_context.py b/runtime/src/devflow_temporal/delivery_pause_context.py index ec27eb49..ddb65268 100644 --- a/runtime/src/devflow_temporal/delivery_pause_context.py +++ b/runtime/src/devflow_temporal/delivery_pause_context.py @@ -1,7 +1,21 @@ -"""Recognize only the queued-before-start pause protocol.""" +"""Recognize admitted pause protocols and carry workflow handoff identity.""" +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass from pathlib import Path +@dataclass(frozen=True) +class PauseContext: + run_id: str + workflow_id: str + execution_run_id: str + epochs: dict[str, int] + + +_CURRENT: ContextVar[PauseContext | None] = ContextVar("delivery_pause_context", default=None) + + def enabled(spec): if "pause_control_version" not in spec: return False @@ -37,3 +51,68 @@ def dispatch_enabled(spec): if enabled(spec): raise ValueError("running-workflow pause is not supported") return intake_enabled(spec) + + +def split_activity_request(value, info): + """Remove the workflow's epoch carrier before stable effect digests are made.""" + if not isinstance(value, dict) or not isinstance(value.get("spec"), dict): + return value, None + spec = value["spec"] + if not enabled(spec) or "_pause_epochs" not in value: + return value, None + epochs = value["_pause_epochs"] + if (not isinstance(epochs, dict) or not epochs + or any(not isinstance(root, str) or not root or type(epoch) is not int or epoch < 0 + for root, epoch in epochs.items()) + or not isinstance(spec.get("run_id"), str) or not spec["run_id"] + or not isinstance(info.workflow_id, str) or not info.workflow_id + or not isinstance(info.workflow_run_id, str) or not info.workflow_run_id): + raise ValueError("invalid pause activity authority") + request = dict(value) + del request["_pause_epochs"] + return request, PauseContext(spec["run_id"], info.workflow_id, + info.workflow_run_id, dict(epochs)) + + +@contextmanager +def activate(context): + token = _CURRENT.set(context) + try: + yield + finally: + _CURRENT.reset(token) + + +def publication_handoff(owner, step): + """Fence an ordinary publication subprocess at its actual launch boundary.""" + from .delivery_execution_authority import activity_authority, require_current + from .delivery_pause import launch_admission_graph + + spec, store = owner.broker.spec, owner.broker.store + binding = _CURRENT.get() + if (not enabled(spec) or binding is None or binding.run_id != spec["run_id"] + or spec.get("feature_delivery") or spec.get("feature_worker") + or set(binding.epochs) != {spec["run_id"]}): + raise ValueError("ordinary publication lacks its complete pause actor binding") + authority = activity_authority() + if (authority["run_id"] != binding.run_id + or authority["workflow_id"] != binding.workflow_id + or authority["execution_run_id"] != binding.execution_run_id): + raise ValueError("publication pause actor differs from its SDK authority") + + def validate(db, epochs, workflow_id, execution_run_id, generation): + if (set(epochs) != {spec["run_id"]} or workflow_id != authority["workflow_id"] + or execution_run_id != authority["execution_run_id"] + or generation != authority["generation"]): + raise ValueError("publication pause graph changed") + require_current(db, spec, authority) + actor = db.execute("""SELECT actor_run_id FROM delivery_pause_actors + WHERE root_id=? AND workflow_id=?""", (spec["run_id"], workflow_id)).fetchone() + if actor is None or actor["actor_run_id"] != spec["run_id"]: + raise ValueError("publication pause actor changed") + + return launch_admission_graph( + store, binding.epochs, owner.key, step, workflow_id=binding.workflow_id, + execution_run_id=binding.execution_run_id, owner_generation=authority["generation"], + validate_graph=validate, + ) diff --git a/runtime/src/devflow_temporal/delivery_publication_execution.py b/runtime/src/devflow_temporal/delivery_publication_execution.py index b255e1c9..acff51f4 100644 --- a/runtime/src/devflow_temporal/delivery_publication_execution.py +++ b/runtime/src/devflow_temporal/delivery_publication_execution.py @@ -117,11 +117,14 @@ def check(self, db=None): raise ValueError("publication executor no longer owns this generation") return row - def state(self, operation): + def mutation(self, operation): row = self.check() - return json.loads(row["mutations_json"]).get(operation, {}).get("state") + return json.loads(row["mutations_json"]).get(operation, {}) + + def state(self, operation): + return self.mutation(operation).get("state") - def _record(self, operation, state): + def _record(self, operation, state, *, evidence=None): with self.broker.store._connect() as db: db.execute("BEGIN IMMEDIATE") row = self.check(db) @@ -129,6 +132,8 @@ def _record(self, operation, state): mutations[operation] = { "state": state, "generation": self.generation, "owner": self.token, } + if evidence is not None: + mutations[operation]["evidence"] = evidence db.execute( "UPDATE delivery_publication_executors SET mutations_json=? WHERE effect_key=?", (canonical_json(mutations), self.key), @@ -145,7 +150,7 @@ def permitted(self, operation): and json.loads(row["mutations_json"]).get(operation, {}).get("state") in {None, "not_started"}) - def mutate(self, operation, function): + def mutate(self, operation, function, *, observe_completion=None): from .delivery_writer_custody import require_writer require_writer(self.broker.spec, store=self.broker.store) @@ -158,12 +163,24 @@ def mutate(self, operation, function): # Popen did not launch. No CLI exit code or diagnostic text grants this. self._record(operation, "not_started") raise - except BaseException: + except BaseException as exc: + from .delivery_pause import PauseDeferred + + if isinstance(exc, PauseDeferred): + # The pause admission rejected the exact handoff before Popen. + self._record(operation, "not_started") + raise self._record(operation, "unknown") raise finally: _MUTATION.reset(token) - self._record(operation, "complete") + try: + evidence = observe_completion() if observe_completion is not None else None + except BaseException: + # The mutation launched; failed completion readback is not a no-launch proof. + self._record(operation, "unknown") + raise + self._record(operation, "complete", evidence=evidence) return result def require(self, operation): diff --git a/runtime/tests/test_delivery_publication_execution.py b/runtime/tests/test_delivery_publication_execution.py index 3c8b4fff..526acb7e 100644 --- a/runtime/tests/test_delivery_publication_execution.py +++ b/runtime/tests/test_delivery_publication_execution.py @@ -71,6 +71,31 @@ def event(kind): return store, spec, broker, checked, tmp_path, receipt +def running_publication_authority(store, spec): + """Install a controlled SDK actor for the still-disabled running protocol.""" + from devflow_temporal import delivery_execution_authority as execution + from devflow_temporal import delivery_pause_context as context + from devflow_temporal.contracts import canonical_json + + run_id = spec['run_id'] + workflow_id = 'delivery-' + run_id + spec['pause_control_version'] = 1 + with store._connect() as db: + db.execute('UPDATE delivery_runs SET request_json=? WHERE run_id=?', + (canonical_json(spec), run_id)) + reservation = execution.reserve(store, store.pending_starts()[0], spec) + authority = execution.bind(store, reservation, 'fixture-sdk-run') + with store._connect() as db: + db.execute("INSERT INTO delivery_pause_control VALUES " + "(?,'running',0,0,?,'fixture-sdk-run',1,NULL,?)", + (run_id, workflow_id, '2026-10-29T00:00:00+00:00')) + db.execute('INSERT INTO delivery_pause_actors VALUES (?,?,?,1,?)', + (run_id, workflow_id, 'fixture-sdk-run', run_id)) + db.execute("INSERT INTO delivery_pause_actor_status VALUES (?,?,'running')", + (run_id, workflow_id)) + return authority, context.PauseContext(run_id, workflow_id, 'fixture-sdk-run', {run_id: 0}) + + def events(root): path = root / 'events.jsonl' return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] @@ -179,3 +204,383 @@ async def test_child_cli_keeps_publication_lease_when_python_owner_is_killed(pub child.kill() child.wait(timeout=5) await until(root, 'create-end') + + +def test_publication_handoff_preserves_staged_work_across_a_pause(publication, monkeypatch): + """A fenced commit cannot start and a later continuation does not repeat staging.""" + import threading + from contextlib import contextmanager + from types import SimpleNamespace + + from temporalio.exceptions import ApplicationError + from temporalio.worker import ExecuteActivityInput + + from devflow_temporal import delivery_execution_authority as execution + from devflow_temporal import delivery_pause as pause + from devflow_temporal import delivery_pause_context as context + from devflow_temporal.delivery_broker import BrokerReadbackUnavailable + from devflow_temporal.delivery_config import DeliveryConfig + from devflow_temporal.delivery_store import DeliveryStore + + store, spec, broker, candidate, root, _receipt = publication + run_id = spec['run_id'] + workflow_id = 'delivery-' + run_id + # Public running-pause admission remains disabled; SDK history is controlled. + authority, binding = running_publication_authority(store, spec) + original_handoff = context.publication_handoff + reached = [] + entered, accepted = threading.Event(), threading.Event() + + def request_pause(): + entered.set() + with pause.root_mutex(store, run_id), store._connect() as db: + db.execute('BEGIN IMMEDIATE') + db.execute("UPDATE delivery_pause_control SET state='pause_requested'," + "revision=1,epoch=1 WHERE root_id=?", (run_id,)) + accepted.set() + + @contextmanager + def pause_after_stage(owner, step): + requester = None + try: + with original_handoff(owner, step) as handoff: + if step == 'stage': + # The concurrent request must wait until Popen has been handed + # off under the shared root mutex, before it can close the fence. + requester = threading.Thread(target=request_pause) + requester.start() + assert entered.wait(5) + assert not accepted.wait(.1) + yield handoff + reached.append(step) + if requester is not None: + assert accepted.wait(5) + finally: + if requester is not None: + requester.join(5) + assert not requester.is_alive() + + monkeypatch.setattr(context, 'publication_handoff', pause_after_stage) + token = execution._ACTIVITY_AUTHORITY.set(authority) + original_head = _git(broker.checkout, 'rev-parse', 'HEAD') + # Exercise the actual interceptor and activity; SDK selection is controlled, + # while the persisted actor and publication handoff checks remain real. + class InvokeActivity: + async def execute_activity(self, input): + return await input.fn(*input.args) + + info = SimpleNamespace(workflow_type='DevflowDeliveryWorkflow', workflow_id=workflow_id, + workflow_run_id='fixture-sdk-run') + monkeypatch.setattr(delivery_activities.activity, 'info', lambda: info) + + authenticated_calls = [] + + async def authenticated(_store, _spec): + authenticated_calls.append(_spec['run_id']) + return authority + + monkeypatch.setattr(execution, 'authenticate_activity', authenticated) + inbound = delivery_activities._DatabaseContentionInbound(InvokeActivity()) + request = {'spec': spec, 'iteration': 0, 'candidate': candidate, + '_pause_epochs': {run_id: 0}} + input = ExecuteActivityInput(fn=delivery_activities.delivery_publish, args=[request], + executor=None, headers={}) + try: + with pytest.raises(ApplicationError) as denied: + asyncio.run(inbound.execute_activity(input)) + assert authenticated_calls == [run_id] + assert denied.value.type == 'PauseDeferred' and denied.value.non_retryable + assert denied.value.details == ({'root_id': run_id, 'operation_key': 'publish:run-1:0', + 'step': 'commit', 'epoch': 1},) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert _git(broker.checkout, 'diff', '--cached', '--name-only') == 'README.md' + assert events(root) == [] + with store._connect() as db: + mutations = json.loads(db.execute( + 'SELECT mutations_json FROM delivery_publication_executors').fetchone()[0]) + assert mutations['stage']['state'] == 'complete' + assert len(mutations['stage']['evidence']['stage_sha256']) == 64 + assert mutations['commit']['state'] == 'not_started' + assert db.execute("SELECT COUNT(*) FROM delivery_pause_denials " + "WHERE root_id=? AND step='commit'", (run_id,)).fetchone()[0] == 1 + + # Reconstruct the store; a new process cannot treat the denial as a + # completed commit or launch the old epoch while the fence is closed. + reopened = DeliveryStore(DeliveryConfig.load(store.config.path)) + reopened_broker = DeliveryBroker(reopened, spec) + with context.activate(binding), pytest.raises(pause.PauseDeferred): + reopened_broker.publish(0, candidate) + assert events(root) == [] + with pause.root_mutex(reopened, run_id), reopened._connect() as db: + db.execute('BEGIN IMMEDIATE') + # Stand in for the future authenticated checkpoint/resume owner. + db.execute("UPDATE delivery_pause_control SET state='running',revision=2 " + "WHERE root_id=?", (run_id,)) + resumed = context.PauseContext(run_id, workflow_id, 'fixture-sdk-run', {run_id: 1}) + + # A completed staging step is not authority to commit a different + # index, even if the worktree still has the original candidate bytes. + original_blob = _git(broker.checkout, 'rev-parse', ':README.md') + altered = root / 'altered-staged-content' + altered.write_text('Unexpected staged content\n') + altered_blob = _git(broker.checkout, 'hash-object', '-w', str(altered)) + _git(broker.checkout, 'update-index', '--cacheinfo', '100644', altered_blob, 'README.md') + try: + with context.activate(resumed), pytest.raises(ValueError, match='staged publication'): + reopened_broker.publish(0, candidate) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + _git(broker.checkout, 'update-index', '--cacheinfo', + '100644', original_blob, 'README.md') + + # A staged deletion leaves the file untracked, so the ordinary unstaged + # diff is empty even though committing it would drop reviewed content. + _git(broker.checkout, 'update-index', '--force-remove', 'README.md') + try: + assert (broker.checkout / 'README.md').is_file() + assert not _git(broker.checkout, 'diff', '--name-only') + with context.activate(resumed), pytest.raises(ValueError, match='staged publication'): + reopened_broker.publish(0, candidate) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + _git(broker.checkout, 'update-index', '--add', '--cacheinfo', + '100644', original_blob, 'README.md') + + # Assume-unchanged also hides mismatched staged bytes from git diff. + _git(broker.checkout, 'update-index', '--cacheinfo', '100644', altered_blob, 'README.md') + _git(broker.checkout, 'update-index', '--assume-unchanged', 'README.md') + try: + assert not _git(broker.checkout, 'diff', '--name-only') + with context.activate(resumed), pytest.raises(ValueError, match='staged publication'): + reopened_broker.publish(0, candidate) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + _git(broker.checkout, 'update-index', '--no-assume-unchanged', 'README.md') + _git(broker.checkout, 'update-index', '--cacheinfo', + '100644', original_blob, 'README.md') + + real_create = reopened_broker._create_owned_pr + first_create = True + + def create(base): + nonlocal first_create + if first_create: + first_create = False + with monkeypatch.context() as missing: + missing.setenv('PATH', str(root / 'no-executables')) + return real_create(base) + return real_create(base) + + monkeypatch.setattr(reopened_broker, '_create_owned_pr', create) + with context.activate(resumed): + with pytest.raises(FileNotFoundError): + reopened_broker.publish(0, candidate) + with reopened._connect() as db: + assert db.execute("SELECT COUNT(*) FROM delivery_pause_steps " + "WHERE root_id=? AND step='create'", (run_id,)).fetchone()[0] == 0 + # A real launched create with a lost outcome is not a no-launch + # failure. It must be read back before any new create is allowed. + monkeypatch.setenv('PUBLICATION_FAIL', '1') + with pytest.raises(BrokerReadbackUnavailable): + reopened_broker.reconcile_publish(0, candidate, create_missing=True) + monkeypatch.delenv('PUBLICATION_FAIL') + assert reopened_broker.reconcile_publish(0, candidate, create_missing=True) == { + 'state': 'pending', 'reason': 'original_create_readback'} + assert [item['kind'] for item in events(root)].count('create-start') == 1 + head = _git(broker.checkout, 'rev-parse', 'HEAD') + # Controlled remote readback finds the original PR after the lost reply. + (root / 'receipt.json').write_text(json.dumps([dict(_receipt[0], headRefOid=head)])) + result = reopened_broker.reconcile_publish(0, candidate, create_missing=True) + assert result['number'] == 1 + assert reached.count('stage') == 1 + assert [item['kind'] for item in events(root)].count('create-start') == 1 + finally: + execution._ACTIVITY_AUTHORITY.reset(token) + + +def test_unknown_stage_readback_never_relaunches_or_commits(publication, monkeypatch): + """A completed Git add with unavailable index proof stays an unknown handoff.""" + from devflow_temporal import delivery_broker as broker_module + from devflow_temporal import delivery_execution_authority as execution + from devflow_temporal import delivery_pause_context as context + from devflow_temporal.delivery_config import DeliveryConfig + from devflow_temporal.delivery_store import DeliveryStore + + store, spec, broker, candidate, root, _receipt = publication + authority, binding = running_publication_authority(store, spec) + original_head = _git(broker.checkout, 'rev-parse', 'HEAD') + original_observe = broker_module._staged_index_evidence + + observations = [] + + def unavailable(checkout): + observations.append(True) + if len(observations) == 1: + return original_observe(checkout) + raise OSError('controlled index readback failure') + + token = execution._ACTIVITY_AUTHORITY.set(authority) + try: + # Git can otherwise ignore reviewed worktree bytes during the first add. + _git(broker.checkout, 'update-index', '--assume-unchanged', 'README.md') + try: + with context.activate(binding), pytest.raises(ValueError, match='index semantics'): + broker.publish(0, candidate) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + _git(broker.checkout, 'update-index', '--no-assume-unchanged', 'README.md') + monkeypatch.setattr(broker_module, '_staged_index_evidence', unavailable) + with context.activate(binding), pytest.raises(OSError, match='index readback'): + broker.publish(0, candidate) + with store._connect() as db: + mutations = json.loads(db.execute( + 'SELECT mutations_json FROM delivery_publication_executors').fetchone()[0]) + assert mutations['stage']['state'] == 'unknown' + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + monkeypatch.setattr(broker_module, '_staged_index_evidence', original_observe) + reopened = DeliveryStore(DeliveryConfig.load(store.config.path)) + with context.activate(binding): + assert DeliveryBroker(reopened, spec).publish(0, candidate) == { + 'state': 'pending', 'reason': 'original_stage_readback'} + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + execution._ACTIVITY_AUTHORITY.reset(token) + + +def test_already_staged_deletion_is_bound_before_pause(publication, monkeypatch): + """A deletion with no remaining stageable path still has an exact index seal.""" + from contextlib import contextmanager + + from devflow_temporal import delivery_execution_authority as execution + from devflow_temporal import delivery_pause as pause + from devflow_temporal import delivery_pause_context as context + + store, spec, broker, _candidate, root, _receipt = publication + original_blob = _git(broker.checkout, 'rev-parse', 'HEAD:README.md') + (broker.checkout / 'README.md').unlink() + _git(broker.checkout, 'add', '-u', '--', 'README.md') + candidate = broker.candidate() + original_head = _git(broker.checkout, 'rev-parse', 'HEAD') + authority, binding = running_publication_authority(store, spec) + original_handoff = context.publication_handoff + + @contextmanager + def close_after_stage(owner, step): + with original_handoff(owner, step) as handoff: + yield handoff + if step == 'stage': + with pause.root_mutex(store, spec['run_id']), store._connect() as db: + db.execute('BEGIN IMMEDIATE') + db.execute("UPDATE delivery_pause_control SET state='pause_requested'," + "revision=1,epoch=1 WHERE root_id=?", (spec['run_id'],)) + + monkeypatch.setattr(context, 'publication_handoff', close_after_stage) + token = execution._ACTIVITY_AUTHORITY.set(authority) + try: + with context.activate(binding), pytest.raises(pause.PauseDeferred) as denied: + broker.publish(0, candidate) + assert denied.value.step == 'commit' + with store._connect() as db: + mutations = json.loads(db.execute( + 'SELECT mutations_json FROM delivery_publication_executors').fetchone()[0]) + assert len(mutations['stage']['evidence']['stage_sha256']) == 64 + db.execute("UPDATE delivery_pause_control SET state='running',revision=2 " + "WHERE root_id=?", (spec['run_id'],)) + resumed = context.PauseContext(spec['run_id'], binding.workflow_id, + binding.execution_run_id, {spec['run_id']: 1}) + # Reintroducing the old blob would discard the reviewed deletion. + _git(broker.checkout, 'update-index', '--add', '--cacheinfo', + '100644', original_blob, 'README.md') + try: + with context.activate(resumed), pytest.raises(ValueError, match='staged publication'): + broker.publish(0, candidate) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + _git(broker.checkout, 'update-index', '--force-remove', 'README.md') + with context.activate(resumed): + result = broker.publish(0, candidate) + assert result['number'] == 1 + assert not _git(broker.checkout, 'ls-tree', '--name-only', 'HEAD', 'README.md') + assert [item['kind'] for item in events(root)].count('create-start') == 1 + finally: + execution._ACTIVITY_AUTHORITY.reset(token) + + +def test_child_handoff_stays_fenced_when_only_parent_resumes(service, tmp_path): + """A parent resume cannot authorize a child that retains its own pause token.""" + from devflow_temporal import delivery_pause as pause + from devflow_temporal.delivery_config import DeliveryConfig + from devflow_temporal.delivery_store import DeliveryStore + + store, request = service + store.submit(request) + parent, child = request['run_id'], 'fixture-child' + workflow, sdk_run = 'fixture-child-workflow', 'fixture-child-sdk-run' + with store._connect() as db: + for root in (parent, child): + db.execute("INSERT INTO delivery_pause_control VALUES " + "(?,'running',0,0,?,?,1,NULL,?)", + (root, workflow, sdk_run, '2026-10-29T00:00:00+00:00')) + db.execute('INSERT INTO delivery_pause_actors VALUES (?,?,?,1,?)', + (root, workflow, sdk_run, child)) + db.execute("INSERT INTO delivery_pause_actor_status VALUES (?,?,'running')", + (root, workflow)) + db.execute("UPDATE delivery_pause_control SET state='paused',revision=1,epoch=1 " + "WHERE root_id=?", (child,)) + + def validate(db, roots, actor_workflow, actor_run, generation): + assert db.in_transaction + if (set(roots) != {parent, child} or actor_workflow != workflow + or actor_run != sdk_run or generation != 1): + raise ValueError('fixture graph identity changed') + actors = db.execute('SELECT root_id,actor_run_id FROM delivery_pause_actors ' + 'WHERE workflow_id=?', (workflow,)).fetchall() + if {(row['root_id'], row['actor_run_id']) for row in actors} != { + (parent, child), (child, child)}: + raise ValueError('fixture graph membership changed') + + def admit(current, bindings): + return pause.launch_admission_graph( + current, bindings, 'child-work', 'launch', workflow_id=workflow, + execution_run_id=sdk_run, owner_generation=1, validate_graph=validate) + + marker = tmp_path / 'child-launched' + with pytest.raises(pause.PauseDeferred) as denied: + with admit(store, {parent: 0, child: 0}): + pytest.fail('independently paused child was launched') + assert denied.value.root_id == child and not marker.exists() + with store._connect() as db: + db.execute("UPDATE delivery_pause_control SET state='paused',epoch=1 WHERE root_id=?", + (parent,)) + # Parent resume is independent of the child's closed token. + db.execute("UPDATE delivery_pause_control SET state='running' WHERE root_id=?", (parent,)) + reopened = DeliveryStore(DeliveryConfig.load(store.config.path)) + with pytest.raises(pause.PauseDeferred) as denied: + with admit(reopened, {parent: 1, child: 1}): + pytest.fail('parent resume cleared the child pause') + assert denied.value.root_id == child and not marker.exists() + with reopened._connect() as db: + db.execute("UPDATE delivery_pause_control SET state='running' WHERE root_id=?", (child,)) + db.execute("UPDATE delivery_pause_actor_status SET state='resume_pending' " + "WHERE root_id=? AND workflow_id=?", (child, workflow)) + with pytest.raises(pause.PauseDeferred): + with admit(reopened, {parent: 1, child: 1}): + pytest.fail('child launch preceded its resume acknowledgement') + with reopened._connect() as db: + db.execute("UPDATE delivery_pause_actor_status SET state='running' " + "WHERE root_id=? AND workflow_id=?", (child, workflow)) + with admit(reopened, {parent: 1, child: 1}): + process = subprocess.Popen([sys.executable, '-c', + 'from pathlib import Path; import sys; ' + 'Path(sys.argv[1]).write_text("started")', str(marker)]) + assert process.wait(timeout=5) == 0 and marker.read_text() == 'started' + with pytest.raises(ValueError, match='already admitted'): + with admit(reopened, {parent: 1, child: 1}): + pytest.fail('same unknown handoff was launched twice') From c12c12aff636f6f5d65c109158fc02af92620261 Mon Sep 17 00:00:00 2001 From: Eloi Date: Sun, 1 Nov 2026 19:21:57 +0100 Subject: [PATCH 23/24] fix(pause): reconcile known command refusals --- .../delivery_operation_receipts.py | 34 +++- .../devflow_temporal/delivery_operations.py | 17 ++ .../src/devflow_temporal/delivery_pause.py | 144 ++++++++++++++-- .../devflow_temporal/delivery_pause_api.py | 13 +- .../test_delivery_pause_intake_public.py | 158 +++++++++++++++++- 5 files changed, 345 insertions(+), 21 deletions(-) diff --git a/runtime/src/devflow_temporal/delivery_operation_receipts.py b/runtime/src/devflow_temporal/delivery_operation_receipts.py index 2ad68bdf..caeee665 100644 --- a/runtime/src/devflow_temporal/delivery_operation_receipts.py +++ b/runtime/src/devflow_temporal/delivery_operation_receipts.py @@ -4,7 +4,7 @@ import json -from .contracts import digest +from .contracts import canonical_json, digest _PHASES = { "recover-publication": {"publication_recovery_queued"}, @@ -20,6 +20,38 @@ } +def settle_pause_refusal(store, operation, timestamp): + """Settle only an exact operation bound to durable no-effect owner evidence.""" + from .delivery_pause import _command_identity, refusal_from_row + + kind, run_id, payload = operation["kind"], operation["run_id"], operation["request"] + if (kind not in {"pause", "resume"} or _command_identity(run_id, kind, payload) is None + or operation.get("domain_command_id") != payload["command_id"] + or operation.get("request_digest") != digest({ + "run_id": run_id, "kind": kind, "request": payload})): + return False + with store._connect() as db: + db.execute("BEGIN IMMEDIATE") + current = db.execute("SELECT * FROM delivery_operations WHERE operation_id=?", + (operation["operation_id"],)).fetchone() + if (current is None or current["state"] not in {"running", "unknown"} + or current["owner_generation"] != operation.get("owner_generation") + or current["kind"] != kind or current["run_id"] != run_id + or current["domain_command_id"] != payload["command_id"] + or current["request_digest"] != operation["request_digest"] + or canonical_json(json.loads(current["request_json"])) != canonical_json(payload)): + return False + row = db.execute("SELECT * FROM delivery_pause_commands WHERE command_id=?", + (payload["command_id"],)).fetchone() + refusal = refusal_from_row(row, run_id, kind, payload) + if refusal is None: + return False + db.execute("""UPDATE delivery_operations SET state='refused',updated_at=?,finished_at=?, + progress='refused',result_json=NULL,error=? WHERE operation_id=?""", + (timestamp, timestamp, refusal, operation["operation_id"])) + return True + + def receipt(store, operation): kind, run_id, payload = operation["kind"], operation["run_id"], operation["request"] command_id = payload.get("command_id") diff --git a/runtime/src/devflow_temporal/delivery_operations.py b/runtime/src/devflow_temporal/delivery_operations.py index cadb3695..d957e25f 100644 --- a/runtime/src/devflow_temporal/delivery_operations.py +++ b/runtime/src/devflow_temporal/delivery_operations.py @@ -12,6 +12,7 @@ import json import os import re +import sqlite3 import uuid from datetime import UTC, datetime from pathlib import Path @@ -356,6 +357,14 @@ async def invoke(): if rejection is not None: _set(service.store, operation_id, "refused", error=rejection) return + if operation["kind"] in {"pause", "resume"} and exc.status_code == 409: + from .delivery_operation_receipts import settle_pause_refusal + + try: + if settle_pause_refusal(service.store, operation, now()): + return + except (ValueError, TypeError, KeyError, OSError, sqlite3.Error): + pass # Missing or invalid owner evidence is still unknown. state = ( "refused" if exc.status_code in (400, 403, 404, 422) @@ -562,6 +571,14 @@ async def reconcile_operation(request: Request, operation_id: str): return operation if operation["state"] != "unknown": return operation + if operation["kind"] in {"pause", "resume"}: + from .delivery_operation_receipts import settle_pause_refusal + + try: + if await asyncio.to_thread(settle_pause_refusal, service.store, operation, now()): + return await asyncio.to_thread(read, service.store, operation_id) + except (ValueError, TypeError, KeyError, OSError, sqlite3.Error): + pass # A changed or unavailable owner receipt cannot prove refusal. payload, run_id, kind = ( operation["request"], operation["run_id"], diff --git a/runtime/src/devflow_temporal/delivery_pause.py b/runtime/src/devflow_temporal/delivery_pause.py index 95504816..4f9b62db 100644 --- a/runtime/src/devflow_temporal/delivery_pause.py +++ b/runtime/src/devflow_temporal/delivery_pause.py @@ -70,6 +70,111 @@ def initialize(db): initialize_intake_pump(db) +_REFUSALS = { + "scope": "pause control scope is not enabled by this service", + "epoch": "invalid pause epoch", + "revision": "invalid control revision", + "conflict": "pause command ID reused with different inputs", + "pause_state": "pause control state or revision changed", + "start_admitted": "intake pause is unavailable after start admission", + "resume_state": "resume control state, revision or epoch changed", + "checkpoint": "checkpoint requires external effect reconciliation before resume", + "clock": "pause workflow clock start acknowledgements are pending", +} + + +class PauseRefused(ValueError): + """A named rejection established before this pause command changed state.""" + + def __init__(self, reason): + self.reason = reason + super().__init__(_REFUSALS[reason]) + + +def _command_digest(root_id, action, body): + request = {"action": action, "root_id": root_id, + "expected_revision": body["expected_revision"]} + if action == "resume": + request["epoch"] = body["epoch"] + return digest(request) + + +def _command_identity(root_id, action, body): + keys = {"command_id", "expected_revision"} | ({"epoch"} if action == "resume" else set()) + if (action not in {"pause", "resume"} or not isinstance(body, dict) or set(body) != keys + or not isinstance(root_id, str) or not root_id or len(root_id) > 512 + or not isinstance(body["command_id"], str) or not body["command_id"] + or len(body["command_id"]) > 512): + return None + return _command_digest(root_id, action, body) + + +def _authenticated_command(row): + """Check the identity available in a historical success or refusal.""" + response = json.loads(row["response_json"]) + if not isinstance(response, dict): + return False + if response.get("state") == "refused": + action, body = response.get("action"), response.get("command") + return (isinstance(body, dict) and body.get("command_id") == row["command_id"] + and _command_identity(row["root_id"], action, body) == row["request_digest"] + and refusal_from_row(row, row["root_id"], action, body) is not None) + if (set(response) != {"root_id", "state", "revision", "epoch"} + or response["root_id"] != row["root_id"] + or response["state"] not in {"pause_requested", "resuming"} + or type(response["revision"]) is not int or response["revision"] < 1 + or type(response["epoch"]) is not int or response["epoch"] < 0 + or (response["state"] == "pause_requested" and response["epoch"] < 1)): + return False + action = "pause" if response["state"] == "pause_requested" else "resume" + body = {"command_id": row["command_id"], + "expected_revision": response["revision"] - 1} + if action == "resume": + body["epoch"] = response["epoch"] + return _command_identity(row["root_id"], action, body) == row["request_digest"] + + +def refusal_from_row(row, root_id, action, body): + """Read a bound refusal, or the immutable conflicting command occupying its ID.""" + expected = _command_identity(root_id, action, body) + if expected is None or row is None: + return None + if row["request_digest"] != expected: + # An occupied ID is evidence of conflict only if its own retained + # identity authenticates; a damaged digest is not a refusal. + return _REFUSALS["conflict"] if _authenticated_command(row) else None + if row["root_id"] != root_id: + return None + response = json.loads(row["response_json"]) + if (not isinstance(response, dict) + or set(response) != {"state", "root_id", "action", "command", "reason", "digest"} + or response["state"] != "refused" or response["root_id"] != root_id + or response["action"] != action or response["digest"] != expected + or canonical_json(response["command"]) != canonical_json(body) + or response["reason"] not in _REFUSALS): + return None + return _REFUSALS[response["reason"]] + + +def remember_refusal(store, root_id, action, body, reason): + """Reserve an exactly identified no-effect command before returning its refusal.""" + expected = _command_identity(root_id, action, body) + if expected is None or reason not in _REFUSALS: + return None + with root_mutex(store, root_id), store._connect() as db: + db.execute("BEGIN IMMEDIATE") + row = db.execute("SELECT * FROM delivery_pause_commands WHERE command_id=?", + (body["command_id"],)).fetchone() + if row is not None: + # A concurrent successful replay must remain a success; never replace it. + return refusal_from_row(row, root_id, action, body) + response = {"state": "refused", "root_id": root_id, "action": action, + "command": body, "reason": reason, "digest": expected} + db.execute("INSERT INTO delivery_pause_commands VALUES (?, ?, ?, ?)", + (body["command_id"], root_id, expected, canonical_json(response))) + return _REFUSALS[reason] + + class PauseDeferred(RuntimeError): """The named step did not start because the owner's pause fence is closed.""" @@ -144,7 +249,7 @@ def request_pause(store, root_id, command_id, expected_revision, *, intake_only= raise ValueError("running-workflow pause is not supported") _nonempty(command_id, "command ID") if type(expected_revision) is not int or expected_revision < 0: - raise ValueError("invalid control revision") + raise PauseRefused("revision") request_digest = digest({"action": "pause", "root_id": root_id, "expected_revision": expected_revision}) with root_mutex(store, root_id), store._connect() as db: @@ -153,11 +258,19 @@ def request_pause(store, root_id, command_id, expected_revision, *, intake_only= (command_id,)).fetchone() if prior is not None: if prior["request_digest"] != request_digest: - raise ValueError("pause command ID reused with different inputs") - return json.loads(prior["response_json"]) + raise PauseRefused("conflict") + if not _authenticated_command(prior): + raise ValueError("invalid pause command receipt") + response = json.loads(prior["response_json"]) + if response["state"] == "refused": + body = {"command_id": command_id, "expected_revision": expected_revision} + if refusal_from_row(prior, root_id, "pause", body) is None: + raise ValueError("invalid pause refusal receipt") + raise PauseRefused(response["reason"]) + return response row = _row(db, root_id) if row["revision"] != expected_revision or row["state"] != "running": - raise ValueError("pause control state or revision changed") + raise PauseRefused("pause_state") if intake_only: run = db.execute("""SELECT r.execution_state,o.state AS outbox_state FROM delivery_runs r JOIN delivery_outbox o USING(run_id) @@ -169,7 +282,7 @@ def request_pause(store, root_id, command_id, expected_revision, *, intake_only= "AND step='start' LIMIT 1", (root_id,)).fetchone() or db.execute("SELECT 1 FROM delivery_pause_actors WHERE root_id=? LIMIT 1", (root_id,)).fetchone()): - raise ValueError("intake pause is unavailable after start admission") + raise PauseRefused("start_admitted") revision, epoch = row["revision"] + 1, row["epoch"] + 1 db.execute("""UPDATE delivery_pause_control SET state='pause_requested', revision=?, epoch=?, checkpoint_json=NULL, updated_at=? WHERE root_id=?""", @@ -204,7 +317,7 @@ def begin_resume(store, root_id, command_id, expected_revision, epoch): _nonempty(command_id, "command ID") _epoch(epoch) if type(expected_revision) is not int or expected_revision < 0: - raise ValueError("invalid control revision") + raise PauseRefused("revision") request = {"action": "resume", "root_id": root_id, "expected_revision": expected_revision, "epoch": epoch} request_digest = digest(request) @@ -214,17 +327,26 @@ def begin_resume(store, root_id, command_id, expected_revision, epoch): (command_id,)).fetchone() if prior is not None: if prior["request_digest"] != request_digest: - raise ValueError("pause command ID reused with different inputs") - return json.loads(prior["response_json"]) + raise PauseRefused("conflict") + if not _authenticated_command(prior): + raise ValueError("invalid pause command receipt") + response = json.loads(prior["response_json"]) + if response["state"] == "refused": + body = {"command_id": command_id, "expected_revision": expected_revision, + "epoch": epoch} + if refusal_from_row(prior, root_id, "resume", body) is None: + raise ValueError("invalid pause refusal receipt") + raise PauseRefused(response["reason"]) + return response row = _row(db, root_id) if (row["state"] != "paused" or row["revision"] != expected_revision or row["epoch"] != epoch): - raise ValueError("resume control state, revision or epoch changed") + raise PauseRefused("resume_state") checkpoint = json.loads(row["checkpoint_json"]) if not checkpoint["resumable"]: - raise ValueError("checkpoint requires external effect reconciliation before resume") + raise PauseRefused("checkpoint") if _missing_clock(db, root_id, epoch, finished=False): - raise ValueError("pause workflow clock start acknowledgements are pending") + raise PauseRefused("clock") revision = row["revision"] + 1 db.execute("""UPDATE delivery_pause_control SET state='resuming', revision=?, updated_at=? WHERE root_id=?""", (revision, _now(), root_id)) diff --git a/runtime/src/devflow_temporal/delivery_pause_api.py b/runtime/src/devflow_temporal/delivery_pause_api.py index 902d4bde..61446f1f 100644 --- a/runtime/src/devflow_temporal/delivery_pause_api.py +++ b/runtime/src/devflow_temporal/delivery_pause_api.py @@ -1,6 +1,7 @@ """Public controls for the explicitly admitted queued-intake pause.""" import asyncio import json +import sqlite3 from fastapi import HTTPException, Request @@ -10,7 +11,7 @@ def _admitted(store, root_id): if not intake_enabled(store.submitted_spec(root_id)): - raise ValueError('pause control scope is not enabled by this service') + raise pause.PauseRefused('scope') def status(store, root_id): @@ -79,8 +80,16 @@ async def command(request, run_id, action): if not isinstance(body, dict) or set(body) != keys: raise ValueError('invalid pause command body') if action == 'resume' and (type(body['epoch']) is not int or body['epoch'] < 1): - raise ValueError('invalid pause epoch') + raise pause.PauseRefused('epoch') return await asyncio.to_thread(apply_command, run_id, action, body) + except pause.PauseRefused as exc: + try: + await asyncio.to_thread(pause.remember_refusal, service.store, run_id, + action, body, exc.reason) + except (ValueError, TypeError, OSError, sqlite3.Error): + # A refusal without durable owner evidence remains ambiguous. + pass + raise HTTPException(409, str(exc)) from exc except (ValueError, TypeError) as exc: raise HTTPException(409, str(exc)) from exc diff --git a/runtime/tests/test_delivery_pause_intake_public.py b/runtime/tests/test_delivery_pause_intake_public.py index a9a7d3b4..9d21d55e 100644 --- a/runtime/tests/test_delivery_pause_intake_public.py +++ b/runtime/tests/test_delivery_pause_intake_public.py @@ -53,6 +53,53 @@ async def test_public_intake_pause_refuses_a_previously_admitted_start(service, 'expected_revision': status['revision']}) assert rejected.status_code == 409 assert 'after start admission' in rejected.text + # A legacy direct caller has reserved a different rejected command ID. + legacy = {'command_id': 'late-legacy', 'expected_revision': status['revision']} + assert (await client.post('/api/runs/run-1/pause', headers=headers, + json=legacy)).status_code == 409 + conflict_envelope = {'operation_id': 'late-conflict', 'run_id': 'run-1', 'kind': 'pause', + 'request': {**legacy, 'expected_revision': status['revision'] + 1}} + assert (await client.post('/api/operations', headers=headers, + json=conflict_envelope)).status_code in (200, 202) + envelope = {'operation_id': 'late-op', 'run_id': 'run-1', 'kind': 'pause', + 'request': {'command_id': 'late-pause', + 'expected_revision': status['revision']}} + response = await client.post('/api/operations', headers=headers, json=envelope) + assert response.status_code in (200, 202) + import asyncio + + async with asyncio.timeout(10): + while True: + refused = (await client.get('/api/operations/late-op')).json() + if refused['state'] not in {'pending', 'running'}: + break + await asyncio.sleep(.02) + assert refused['state'] == 'refused' and 'after start admission' in refused['error'] + async with asyncio.timeout(10): + while True: + conflict = (await client.get('/api/operations/late-conflict')).json() + if conflict['state'] not in {'pending', 'running'}: + break + await asyncio.sleep(.02) + assert conflict['state'] == 'refused' and 'different inputs' in conflict['error'] + assert (await client.get('/api/runs/run-1/pause-control')).json()['observed'] == 'running' + # A separate ordinary run has no admitted pause scope at all. + ordinary = {**request, 'run_id': 'run-2', 'work_id': 'work-2', + 'command_id': 'command-2', 'branch': 'feat/fixture-2', + 'issue_url': 'https://github.com/example/fixture/issues/4'} + created = await client.post('/api/runs', headers=headers, json=ordinary) + assert created.status_code == 200, created.text + unsupported = {'operation_id': 'unsupported-op', 'run_id': 'run-2', 'kind': 'pause', + 'request': {'command_id': 'unsupported-cmd', 'expected_revision': 0}} + assert (await client.post('/api/operations', headers=headers, + json=unsupported)).status_code in (200, 202) + async with asyncio.timeout(10): + while True: + scope = (await client.get('/api/operations/unsupported-op')).json() + if scope['state'] not in {'pending', 'running'}: + break + await asyncio.sleep(.02) + assert scope['state'] == 'refused' and 'scope is not enabled' in scope['error'] @pytest.mark.asyncio @@ -234,11 +281,14 @@ async def race_before_start(*args, **kwargs): async def test_public_pause_operations_reconcile_lost_acknowledgements(unstarted, monkeypatch): """A lost response never repeats a fenced command or fabricates its outcome.""" import asyncio + import json + from contextlib import contextmanager from test_delivery_pause_intake import begin_intake_settlement, controlled_no_start from devflow_temporal import delivery_pause as pause from devflow_temporal import delivery_pause_intake as intake + from devflow_temporal import delivery_pause_namespace _controlled_cutover(monkeypatch) store, spec, run = unstarted @@ -248,12 +298,18 @@ async def test_public_pause_operations_reconcile_lost_acknowledgements(unstarted calls = [] original_pause = pause.request_pause - def lost_pause(*args, **kwargs): + def tracked_pause(*args, **kwargs): calls.append('pause') - original_pause(*args, **kwargs) - raise OSError('fixture lost acknowledgement after commit') + return original_pause(*args, **kwargs) + + monkeypatch.setattr(pause, 'request_pause', tracked_pause) + original_scope = delivery_pause_namespace.intake_namespace_scope - monkeypatch.setattr(pause, 'request_pause', lost_pause) + @contextmanager + def fail_on_scope_exit(config): + with original_scope(config): + yield + raise ValueError('fixture namespace revalidation failed after commit') async def operation(client, headers, envelope): response = await client.post('/api/operations', headers=headers, json=envelope) @@ -269,9 +325,55 @@ async def operation(client, headers, envelope): session = await client.get('/api/session') headers = {'Origin': origin, 'X-Devflow-CSRF': session.json()['csrf_token']} status = (await client.get(f'/api/runs/{root}/pause-control')).json() + stale = {'operation_id': 'stale-op', 'run_id': root, 'kind': 'pause', + 'request': {'command_id': 'stale-cmd', 'expected_revision': 99}} + refused = await operation(client, headers, stale) + assert refused['state'] == 'refused' and 'revision changed' in refused['error'] + assert (await client.get(f'/api/runs/{root}/pause-control')).json()['observed'] == 'running' + assert (await operation(client, headers, stale))['state'] == 'refused' + alias = await client.post('/api/operations', headers=headers, + json={**stale, 'operation_id': 'stale-alias'}) + assert alias.json()['operation_id'] == 'stale-op' + direct = await client.post(f'/api/runs/{root}/pause', headers=headers, + json=stale['request']) + assert direct.status_code == 409 + conflict = await client.post('/api/operations', headers=headers, + json={**stale, 'operation_id': 'stale-conflict', + 'request': {**stale['request'], 'expected_revision': status['revision']}}) + assert conflict.status_code == 409 + # Simulate loss after the owner saved its no-effect receipt. Changed + # transport identity or a corrupted receipt cannot manufacture a refusal. + with store._connect() as db: + receipt = db.execute("SELECT response_json FROM delivery_pause_commands " + "WHERE command_id='stale-cmd'").fetchone()[0] + db.execute("UPDATE delivery_operations SET state='unknown',request_json=? " + "WHERE operation_id='stale-op'", + (json.dumps({**stale['request'], 'expected_revision': 98}),)) + assert (await client.post('/api/operations/stale-op/reconcile', + headers=headers)).json()['state'] == 'unknown' + with store._connect() as db: + db.execute("UPDATE delivery_operations SET request_json=? " + "WHERE operation_id='stale-op'", (json.dumps(stale['request']),)) + bad_receipt = {**json.loads(receipt), 'root_id': 'substituted'} + db.execute("UPDATE delivery_pause_commands SET response_json=? WHERE command_id=?", + (json.dumps(bad_receipt), 'stale-cmd')) + assert (await client.post('/api/operations/stale-op/reconcile', + headers=headers)).json()['state'] == 'unknown' + assert (await client.post(f'/api/runs/{root}/pause', headers=headers, + json=stale['request'])).status_code == 409 + with store._connect() as db: + db.execute("UPDATE delivery_pause_commands SET response_json=? WHERE command_id=?", + (receipt, 'stale-cmd')) + assert (await client.post('/api/operations/stale-op/reconcile', + headers=headers)).json()['state'] == 'refused' + calls.clear() envelope = {'operation_id': 'pause-op', 'run_id': root, 'kind': 'pause', 'request': {'command_id': 'pause-cmd', 'expected_revision': status['revision']}} + # The owner commits, then namespace revalidation fails with HTTP 409. + # This is ambiguous until the exact successful owner receipt is read. + monkeypatch.setattr(delivery_pause_namespace, 'intake_namespace_scope', fail_on_scope_exit) lost = await operation(client, headers, envelope) + monkeypatch.setattr(delivery_pause_namespace, 'intake_namespace_scope', original_scope) assert lost['state'] == 'unknown' assert (await client.get(f'/api/runs/{root}/pause-control')).json()[ 'observed'] == 'pause_requested' @@ -292,6 +394,16 @@ async def operation(client, headers, envelope): base_url=origin) as client: session = await client.get('/api/session') headers = {'Origin': origin, 'X-Devflow-CSRF': session.json()['csrf_token']} + with store._connect() as db: + saved_digest = db.execute("SELECT request_digest FROM delivery_pause_commands " + "WHERE command_id='pause-cmd'").fetchone()[0] + db.execute("UPDATE delivery_pause_commands SET request_digest='corrupt' " + "WHERE command_id='pause-cmd'") + assert (await client.post('/api/operations/pause-op/reconcile', + headers=headers)).json()['state'] == 'unknown' + with store._connect() as db: + db.execute("UPDATE delivery_pause_commands SET request_digest=? " + "WHERE command_id='pause-cmd'", (saved_digest,)) recovered = (await client.post('/api/operations/pause-op/reconcile', headers=headers)).json() assert recovered['state'] == 'succeeded' @@ -302,6 +414,18 @@ async def operation(client, headers, envelope): intake.seal_unstarted(store, root, 1, run['revision'], prove_unstarted=controlled_no_start(store)) paused = pause.publish_paused(store, root, 1) + invalid = {'operation_id': 'invalid-resume-op', 'run_id': root, 'kind': 'resume', + 'request': {'command_id': 'invalid-resume-cmd', + 'expected_revision': paused['revision'], 'epoch': 0}} + assert (await operation(client, headers, invalid))['state'] == 'refused' + assert (await client.post(f'/api/runs/{root}/resume', headers=headers, + json=invalid['request'])).status_code == 409 + with store._connect() as db: + db.execute("UPDATE delivery_operations SET state='unknown' " + "WHERE operation_id='invalid-resume-op'") + assert (await client.post('/api/operations/invalid-resume-op/reconcile', + headers=headers)).json()['state'] == 'refused' + assert (await client.get(f'/api/runs/{root}/pause-control')).json()['observed'] == 'paused' original_resume = pause.begin_resume def lost_resume(*args, **kwargs): @@ -315,8 +439,6 @@ def lost_resume(*args, **kwargs): 'epoch': 1}} # Simulate a restart after transport persistence but before admission. # Startup must leave this resume pending until the user repeats it. - import json - from devflow_temporal.contracts import digest with store._connect() as db: @@ -357,6 +479,16 @@ def lost_resume(*args, **kwargs): headers=headers)).json() assert substituted['state'] == 'unknown' assert calls == ['pause', 'resume'] + # A successful historical receipt still rejects a conflicting request + # and replays unchanged after state advances. + direct_conflict = await client.post(f'/api/runs/{root}/pause', headers=headers, + json={**envelope['request'], 'expected_revision': status['revision'] + 1}) + assert direct_conflict.status_code == 409 + replay = await client.post(f'/api/runs/{root}/pause', headers=headers, + json=envelope['request']) + assert replay.status_code == 200 and replay.json() == recovered['result'] + current = (await client.get(f'/api/runs/{root}/pause-control')).json() + assert current['observed'] == 'resuming' @pytest.mark.asyncio @@ -422,6 +554,18 @@ async def completed(client, op): while not server.started: await asyncio.sleep(.02) assert (await cli('pause-control'))['scope'] == 'before_start' + stale = {'operation_id': 'cross-stale', 'command_id': 'cross-stale-cmd', + 'expected_revision': 99} + assert (await cli('pause', stale))['operation_id'] == 'cross-stale' + async with httpx.AsyncClient(base_url=host) as client: + assert (await completed(client, 'cross-stale'))['state'] == 'refused' + async with create_connected_server_and_client_session( + build_server(store.config.path) + ) as mcp: + repeated = await mcp.call_tool('pause_run', { + 'run_id': spec['run_id'], 'request_json': json.dumps(stale)}) + assert not repeated.isError + assert json.loads(repeated.content[0].text)['state'] == 'refused' request = {'operation_id': 'cross-pause', 'command_id': 'cross-pause-cmd', 'expected_revision': 0} first = await cli('pause', request) @@ -456,7 +600,7 @@ async def completed(client, op): assert (await cli('pause-control'))['observed'] == 'resuming' with store._connect() as db: assert db.execute('SELECT COUNT(*) FROM delivery_pause_commands ' - 'WHERE root_id=?', (spec['run_id'],)).fetchone()[0] == 2 + 'WHERE root_id=?', (spec['run_id'],)).fetchone()[0] == 3 finally: server.should_exit = True await asyncio.wait_for(serving, 10) From e6cdc074687b5ea0b4e8ce36c66068952bbadaed Mon Sep 17 00:00:00 2001 From: Eloi Date: Sun, 1 Nov 2026 19:32:14 +0100 Subject: [PATCH 24/24] fix(pause): reject unaddressable operations --- .../devflow_temporal/delivery_operations.py | 7 +++ .../test_delivery_pause_intake_public.py | 53 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/runtime/src/devflow_temporal/delivery_operations.py b/runtime/src/devflow_temporal/delivery_operations.py index d957e25f..e20b18f9 100644 --- a/runtime/src/devflow_temporal/delivery_operations.py +++ b/runtime/src/devflow_temporal/delivery_operations.py @@ -463,6 +463,13 @@ async def create_operation(request: Request): raise HTTPException(400, "run_id is required except for submit") if KINDS[kind][1] != "observation" or kind == "issue-context": mutationguard(request) + if kind in {"pause", "resume"}: + from .delivery_pause import _command_identity + + if _command_identity(run_id, kind, payload) is None: + raise HTTPException( + 400, "pause operation requires exact command identity and fields" + ) request_digest = digest({"run_id": run_id, "kind": kind, "request": payload}) command_id = payload.get("command_id") if KINDS[kind][1] == "observation" or not isinstance(command_id, str) or not command_id: diff --git a/runtime/tests/test_delivery_pause_intake_public.py b/runtime/tests/test_delivery_pause_intake_public.py index 9d21d55e..75902a14 100644 --- a/runtime/tests/test_delivery_pause_intake_public.py +++ b/runtime/tests/test_delivery_pause_intake_public.py @@ -506,6 +506,7 @@ async def test_cli_and_mcp_use_the_same_durable_pause_commands(unstarted, monkey from devflow_temporal import delivery_pause as pause from devflow_temporal import delivery_pause_intake as intake + from devflow_temporal.contracts import digest from devflow_temporal.delivery_mcp import build_server _controlled_cutover(monkeypatch) @@ -529,7 +530,7 @@ async def dormant(): server = uvicorn.Server(uvicorn.Config(app, host='127.0.0.1', port=port, log_level='error')) serving = asyncio.create_task(server.serve(sockets=[listener])) - async def cli(command, request=None): + async def cli(command, request=None, *, error=None): argv = [sys.executable, '-m', 'devflow_temporal.delivery_control', '--config', str(store.config.path), command, '--id', spec['run_id']] if request is not None: @@ -538,6 +539,9 @@ async def cli(command, request=None): argv += ['--request', str(request_path)] result = await asyncio.to_thread(subprocess.run, argv, capture_output=True, text=True, timeout=15) + if error is not None: + assert result.returncode != 0 and error in result.stderr + return None assert result.returncode == 0, result.stderr return json.loads(result.stdout) @@ -553,7 +557,52 @@ async def completed(client, op): async with asyncio.timeout(10): while not server.started: await asyncio.sleep(.02) - assert (await cli('pause-control'))['scope'] == 'before_start' + before = await cli('pause-control') + assert before['scope'] == 'before_start' + malformed = [ + ('pause', {'command_id': '', 'expected_revision': 0}), + ('pause', {'command_id': 17, 'expected_revision': 0}), + ('pause', {'command_id': 'x' * 513, 'expected_revision': 0}), + ('pause', {'expected_revision': 0}), + ('pause', {'command_id': 'missing-revision'}), + ('pause', {'command_id': 'extra-pause', 'expected_revision': 0, 'extra': True}), + ('resume', {'command_id': 'missing-epoch', 'expected_revision': 0}), + ('resume', {'command_id': 'extra-resume', 'expected_revision': 0, 'epoch': 1, + 'extra': True}), + ] + with store._connect() as db: + counts = (db.execute('SELECT COUNT(*) FROM delivery_operations').fetchone()[0], + db.execute('SELECT COUNT(*) FROM delivery_pause_commands').fetchone()[0]) + async with httpx.AsyncClient(base_url=host) as client: + session = await client.get('/api/session') + headers = {'Origin': host, 'X-Devflow-CSRF': session.json()['csrf_token']} + for index, (kind, payload) in enumerate(malformed): + response = await client.post('/api/operations', headers=headers, + json={'operation_id': f'malformed-{index}', 'run_id': spec['run_id'], + 'kind': kind, 'request': payload}) + assert response.status_code == 400, response.text + await cli('pause', {'operation_id': 'malformed-cli', 'command_id': '', + 'expected_revision': 0}, error='service HTTP 400') + with store._connect() as db: + assert counts == (db.execute('SELECT COUNT(*) FROM delivery_operations') + .fetchone()[0], + db.execute('SELECT COUNT(*) FROM delivery_pause_commands') + .fetchone()[0]) + assert (await client.get(f"/api/runs/{spec['run_id']}/pause-control")).json() == before + # Existing unaddressable journal rows stay unknown rather than being + # rewritten or treated as proof of a refusal. + payload = {'command_id': '', 'expected_revision': 0} + with store._connect() as db: + db.execute("""INSERT INTO delivery_operations + (operation_id,run_id,kind,stage,request_digest,request_json,state, + created_at,updated_at,progress) + VALUES ('legacy-malformed',?,'pause','execution',?,?,'unknown', + '2026','2026','reconciliation required')""", + (spec['run_id'], digest({'run_id': spec['run_id'], 'kind': 'pause', + 'request': payload}), json.dumps(payload))) + legacy = (await client.post('/api/operations/legacy-malformed/reconcile', + headers=headers)).json() + assert legacy['state'] == 'unknown' stale = {'operation_id': 'cross-stale', 'command_id': 'cross-stale-cmd', 'expected_revision': 99} assert (await cli('pause', stale))['operation_id'] == 'cross-stale'