From 98386ebbfd1e5fd5703d9250e47f8b2cdae2f790 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:18:30 +0100 Subject: [PATCH 1/8] fix(recovery): retain candidates after readiness failures --- docs/trusted-local-delivery.md | 18 + .../devflow_temporal/delivery_activities.py | 12 +- .../delivery_automatic_retry.py | 13 + .../devflow_temporal/delivery_gate_retry.py | 122 ++- .../delivery_readiness_gate_retry.py | 127 +++ .../src/devflow_temporal/delivery_workflow.py | 10 + .../tests/test_delivery_automatic_retry.py | 15 +- ...test_delivery_feature_readiness_handoff.py | 195 +++++ .../test_delivery_readiness_gate_retry.py | 739 ++++++++++++++++++ 9 files changed, 1225 insertions(+), 26 deletions(-) create mode 100644 runtime/src/devflow_temporal/delivery_readiness_gate_retry.py create mode 100644 runtime/tests/test_delivery_feature_readiness_handoff.py create mode 100644 runtime/tests/test_delivery_readiness_gate_retry.py diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index ebee2114..6616d626 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -181,6 +181,24 @@ create its reports before the dedicated browser gate. Nested output parents are created only inside an authenticated checkout. Existing unregistered outputs, tracked contents, protected paths and symlinked parents remain rejected. +A check stopped before launch by execution readiness records a versioned failure +class with its stage, candidate and immutable readiness-receipt digest. The +existing zero-repair gate control can admit at most two fresh readiness retries +for the unchanged candidate, preserving the old receipt, attempt and effect +ledger. Every retry probes readiness again and runs the original frozen gate; +its readiness receipt is never a gate pass. Other failed gates, pending product +findings, uncertain cleanup, changed source or unsettled effects refuse admission. +The initial route is qualified only for the fake fixture provider: native runs +refuse with `native_descendant_settlement_unknown` until a complete descendant +settlement contract can be authenticated. An exhausted retry allowance reports +`readiness_retry_limit_exhausted`. Neither blocker is a launch grant. A transient +unpublished stop with a recorded passing implementation is retained for an +authenticated same-candidate continuation; the automatic fresh-run path refuses +to replace it with another implementation. For a feature worker, the new +readiness path also requires an already authenticated successor assignment. The +current coordinator examines the stopped worker before reserving that assignment, +so an unreserved continuation remains blocked rather than bypassing ownership. + An older run stopped by that browser-output registration defect can use one `published_check_prelaunch_retry` with zero additional implementation turns. Admission requires its exact closed UNKNOWN checkpoint, passed local checks and diff --git a/runtime/src/devflow_temporal/delivery_activities.py b/runtime/src/devflow_temporal/delivery_activities.py index bca456e6..2f37ec2a 100644 --- a/runtime/src/devflow_temporal/delivery_activities.py +++ b/runtime/src/devflow_temporal/delivery_activities.py @@ -1221,12 +1221,13 @@ def _check_readiness(broker, request, attempt): workspace = broker.gate_checkout("verify", request["iteration"], request["candidate"]) requirements = describe_requirements( request["spec"], workspace, request["spec"].get("accepted_plan", "")) - generation = int(digest({"spec": request["spec"], "candidate": request["candidate"]})[:5], - 16) % 300000 - probe_attempt = 10 + generation * 3 + {100: 0, 101: 1, 102: 2}[attempt] + from .delivery_readiness_gate_retry import SLOTS, failure, probe_attempt + + stage = next(name for name, slot in SLOTS.items() if slot == attempt) + selected_attempt = probe_attempt(request["spec"], request["candidate"], attempt) readiness = prepare_and_probe(broker.store, request["spec"], request["candidate"], requirements, "plan", iteration=request["iteration"], - attempt=probe_attempt, workspace=workspace) + attempt=selected_attempt, workspace=workspace) if readiness.get("state") == "passed": verify_receipt(broker.store, request["spec"], request["candidate"], readiness, requirements) return readiness @@ -1237,7 +1238,8 @@ def _check_readiness(broker, request, attempt): "failure_kind": "readiness_preparation", "launched": False, "cleanup": cleanup, "candidate_id": request["candidate"]["id"], "source_unchanged": broker.candidate() == request["candidate"], - "reason": "execution readiness prevented the check", "readiness": readiness} + "reason": "execution readiness prevented the check", "readiness": readiness, + "failure_class": failure(readiness, stage, request["candidate"]["id"], cleanup)} @activity.defn(name="delivery_checks") diff --git a/runtime/src/devflow_temporal/delivery_automatic_retry.py b/runtime/src/devflow_temporal/delivery_automatic_retry.py index 00295fc5..bf5e01ee 100644 --- a/runtime/src/devflow_temporal/delivery_automatic_retry.py +++ b/runtime/src/devflow_temporal/delivery_automatic_retry.py @@ -554,6 +554,19 @@ def retry_once(store, *, stopped=lambda: False): 'SELECT issue_url FROM delivery_runs')) if count >= spec['policy']['max_attempts'] or unavailable(store, db, spec): continue + roles = result.get('roles') + if not isinstance(roles, list) or any(not isinstance(role, dict) for role in roles): + _record_skip(store, row, ValueError( + 'closed run lacks its complete implementation history')) + continue + if any(role.get('role') == 'implement' and role.get('status') == 'pass' + for role in roles): + # A fresh run would start implementation again and abandon the + # stopped candidate. It is not a same-candidate retry grant. + _record_skip(store, row, ValueError( + 'retained implementation requires an authenticated same-candidate ' + 'continuation; inspect the stopped recovery options')) + continue base = fresh_unpublished_base(store, spec) identity = digest({'run_id': spec['run_id'], 'request_digest': row['request_digest']})[:24] diff --git a/runtime/src/devflow_temporal/delivery_gate_retry.py b/runtime/src/devflow_temporal/delivery_gate_retry.py index a68a766e..21d948f5 100644 --- a/runtime/src/devflow_temporal/delivery_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_gate_retry.py @@ -74,8 +74,11 @@ def transient_stop(state): def prepublication_preparation_failed(state): """Recognize the controller's exact preparation stop, never a product finding.""" from .delivery_baseline_contract import preparation_failure + from .delivery_readiness_gate_retry import possible as readiness_possible precheck = state.get('checks', {}).get('prepublish', {}) + if readiness_possible(state) == 'prepublish' and precheck.get('state') == 'failed': + return True # Selection hint only; snapshot authenticates the receipt. prerequisite = preparation_failure(precheck) return (precheck.get('state') == 'failed' and prerequisite is not None and state.get('error') == f'environment preparation failed: {prerequisite}; ' @@ -547,6 +550,11 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= from .delivery_prerequisite_gate_recovery import possible as prerequisite_possible prerequisite_retry = kind == KIND and prerequisite_possible(spec, state) + from .delivery_readiness_gate_retry import possible as readiness_possible + + readiness_stage = readiness_possible(state) + readiness_retry = bool(readiness_stage and ( + (unpublished and readiness_stage == 'prepublish') or kind == KIND)) # A recorded gate verdict stands: a transport or controller stop after it is # never a fresh assessment of the same head. verdict = gate_failed(state.get('checks') or {}) @@ -587,7 +595,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= and previous.get('kind') == PREPUBLICATION_KIND and spec.get('gate_retry_generation') == 1) history = previous - ci_observations = transient_generations = 0 + ci_observations = transient_generations = readiness_generations = 0 published_after_recovery = bool(not unpublished and previous and previous.get('kind') in { PREPUBLICATION_KIND, 'pending_publication_retry', 'repair_continuation', 'published_head_validation'}) @@ -595,6 +603,8 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= 'finalized_checkpoint'): published_after_recovery = False while history: + if history.get('seal', {}).get('readiness_observation'): + readiness_generations += 1 if prerequisite_retry and history.get('seal', {}).get('ordinary_prerequisite_observation'): raise ValueError('ordinary prerequisite already received its one gate recovery') if browser_prelaunch and history.get('kind') == PRELAUNCH_KIND: @@ -613,7 +623,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= if transient_retry and transient_generations >= TRANSIENT_GATE_RETRIES: raise ValueError('this run already received its bounded transient gate retries') if ((renewed or published_after_recovery or prelaunch or report_retry or controller_only) - and spec['provider'] != 'fake'): + and not readiness_retry and spec['provider'] != 'fake'): from .delivery_native_preparation import native_identity if native_identity(spec)['runtime_payload_sha256'] in _consumed_payloads( spec, state, previous, include_implementation=controller_only): @@ -672,6 +682,31 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= browser_artifact_observation = None verification_observation = None prerequisite_observation = None + readiness_observation = None + if readiness_retry: + from .delivery_readiness_gate_retry import observe + + if state_checks != row_checks: + raise ValueError('readiness retry lost its exact original checks') + if spec.get('feature_worker'): + from .delivery_feature_execution import require_worker_handoff + + require_worker_handoff(store, spec) + readiness_observation = observe(spec, state) + latest = next((r for r in reversed(roles) if r.get('role') == 'implement'), None) + matches = [a for a in attempts if latest and a['role'] == 'implement' + and a['iteration'] == row['iteration'] + and a['session_id'] == latest.get('session_id')] + from .delivery_stopped_resume import _implementation_receipt + + if (not latest or latest.get('status') != 'pass' or latest.get('findings') + or latest.get('cleanup') != 'confirmed' or not latest.get('session_id') + or len(matches) != 1 or not _implementation_receipt(spec, latest, matches[0]) + or any((latest.get('candidate') or {}).get(key) != candidate.get(key) + for key in ('content_sha256', 'base_sha', 'environment_digest', + 'policy_digest'))): + raise ValueError('readiness retry lacks its exact passing implementation receipt') + failed_gate = True if prerequisite_retry: from .delivery_prerequisite_gate_recovery import observe @@ -770,7 +805,8 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= or (previous is not None and not ( renewed or published_after_recovery or prelaunch or report_retry or ci_only or transient_retry or attribution_retry - or verification_retry or browser_artifact_retry or prerequisite_retry)) + or verification_retry or browser_artifact_retry or prerequisite_retry + or readiness_retry)) or row['phase'] != 'blocked' or row['outcome'] != 'blocked' or row['execution_state'] != 'blocked' or row['cleanup'] != ('unknown' if prelaunch else 'confirmed') @@ -778,7 +814,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= or (prelaunch and state.get('cleanup') != 'unknown') or (not prelaunch and state.get('cleanup') != 'confirmed' and reconciled is None) or row['error'] != state.get('error') - or (not (transient_retry or transient_ci or masked_exhaustion) + or (not (transient_retry or transient_ci or masked_exhaustion or readiness_retry) and state.get('error') not in ({'repair limit exhausted', 'required CI did not confirm this PR head'} if report_retry @@ -842,7 +878,8 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= ancestor = ancestor.get('original_recovery') if admitted and (not (renewed or published_after_recovery or prelaunch or report_retry or ci_only or transient_retry or attribution_retry - or verification_retry or browser_artifact_retry or prerequisite_retry) + or verification_retry or browser_artifact_retry or prerequisite_retry + or readiness_retry) or (digest(prior_gate) != attribution_observation['prior_gate_digest'] if attribution_retry else prior_gate != ancestor)): raise ValueError('this run already received its bounded gate assessment retry') @@ -854,21 +891,47 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= store, spec, previous, candidate, publication, _inventory_profile) elif _inventory_profile is not None: raise ValueError('inventory physical custody belongs only to zero-model CI') + cleanup = (prerequisite_observation['resources'] if prerequisite_retry + else verification_observation['resources'] if verification_retry + else report_observation['resources'] if layout_report + else historical_teardown or prelaunch_observation or ( + _stopped_cleanup(spec, read_only=True) if _observation + else _stopped_cleanup(spec))) + if readiness_retry: + from .delivery_lifecycle import require_transition + from .delivery_readiness_gate_retry import RETRY_LIMIT + + # An observed native monitor does not prove its complete descendant set. + # The fake fixture has no native actor; real execution needs a separate + # settlement contract before this newly introduced route can launch work. + require_transition({ + 'operation': kind, 'owner': 'delivery_gate_retry', 'owner_proof': True, + 'normalized_lifecycle': 'stopped', + 'proof_digest': digest({'row': _sealed_row(row), 'cleanup': cleanup, + 'readiness': readiness_observation, + 'prior_readiness_retries': readiness_generations}), + 'comparisons': [], 'predicates': [ + {'code': 'native_descendant_settlement_unknown', + 'satisfied': True if spec['provider'] == 'fake' else None, + 'missing_or_unknown_fact': 'complete native descendant settlement', + 'next_action': 'obtain authenticated native descendant settlement'}, + {'code': 'readiness_retry_limit_exhausted', + 'satisfied': readiness_generations < RETRY_LIMIT, + 'missing_or_unknown_fact': 'remaining infrastructure retry allowance', + 'next_action': 'inspect retained failures and reconcile the environment'}, + ], + }, {}) result = {'row': _sealed_row(row), 'closed': closed, 'original_spec': spec, 'attempts': attempts, 'effects': effects, 'candidate': candidate, 'publication': publication, - 'cleanup': (prerequisite_observation['resources'] if prerequisite_retry - else verification_observation['resources'] if verification_retry - else report_observation['resources'] if layout_report - else historical_teardown or prelaunch_observation or ( - _stopped_cleanup(spec, read_only=True) if _observation - else _stopped_cleanup(spec))), + 'cleanup': cleanup, 'work_binding': binding, 'previous_digest': digest(previous) if previous else None, 'prior_gate_digest': digest(prior_gate) if prior_gate else None, **({'local_download_retry': True} if local_download_retry else {}), **({'ordinary_prerequisite_observation': prerequisite_observation} if prerequisite_observation else {}), + **({'readiness_observation': readiness_observation} if readiness_observation else {}), **({'report_observation': report_observation} if report_observation else {}), **({'verification_observation': verification_observation} if verification_observation else {}), @@ -881,13 +944,14 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= if attribution_retry or layout_report or verification_retry else {}), **({'browser_prelaunch_observation': browser_observation} if browser_observation else {}), - 'stage': 'prerequisite' if prerequisite_retry + 'stage': 'readiness' if readiness_retry else 'prerequisite' if prerequisite_retry else 'transient' if transient_retry or verification_retry else 'report' if report_retry else 'ci' if ci_only else 'published' if published_after_recovery or prelaunch or controller_only or attribution_retry or browser_artifact_retry else None, - 'generation': (transient_generations + 1 if transient_retry + 'generation': (readiness_generations + 1 if readiness_retry + else transient_generations + 1 if transient_retry else ci_observations + 1 if ci_only else (2 if previous else 1) if attribution_retry or browser_artifact_retry @@ -940,6 +1004,8 @@ def admit(store, run_id, payload, *, preflight=False): return json.loads(prior['response_json']) seal = snapshot(store, run_id, kind, _observation=preflight) previous = _retained_recovery(store, run_id, seal) + if seal.get('readiness_observation') and 'verification_test_paths' in payload: + raise ValueError('readiness retry must retain the original frozen test selectors') if 'verification_test_paths' in payload: from .delivery_plan_checks import planned_checks @@ -974,7 +1040,7 @@ def admit(store, run_id, payload, *, preflight=False): ordering = (seal.get('browser_artifact_observation') or {}).get( 'cause') == 'browser_artifact_assessment_order' if (preflight_generation or seal.get('preparation_observation') - or layout or verification or ordering) \ + or seal.get('readiness_observation') or layout or verification or ordering) \ and spec.get('feature_worker'): from .delivery_feature_execution import require_worker_handoff @@ -1022,7 +1088,8 @@ def admit(store, run_id, payload, *, preflight=False): execution['implementation_role_home_generation'] = implementation_generation( spec, previous) execution['role_home_generation'] = ( - f'report-retry-{generation}' if seal['stage'] == 'report' + f'readiness-retry-{generation}' if seal['stage'] == 'readiness' + else f'report-retry-{generation}' if seal['stage'] == 'report' else f'transient-gate-retry-{generation}' if seal['stage'] == 'transient' else f'gate-retry-{generation}') execution['gate_retry_generation'] = generation @@ -1071,7 +1138,9 @@ def admit(store, run_id, payload, *, preflight=False): else {**seal['publication'], 'candidate': after})} _immutable(root / 'admission.json', recovery) preserve_resources(root, spec) - workflow_id = (f'delivery-{run_id}-ci-retry-{generation}' if kind == CI_KIND else + workflow_id = (f'delivery-{run_id}-readiness-gates-retry-{generation}' + if seal['stage'] == 'readiness' else + f'delivery-{run_id}-ci-retry-{generation}' if kind == CI_KIND else f'delivery-{run_id}-report-gates-retry-{generation}' if seal['stage'] == 'report' else f'delivery-{run_id}-transient-gates-retry-{generation}' @@ -1093,19 +1162,19 @@ def admit(store, run_id, payload, *, preflight=False): if canonical_json(_sealed_row(row)) != canonical_json(seal['row']): raise ValueError('gate retry admission lost its stopped projection') if (verification or ordering or seal.get('ordinary_prerequisite_observation') - or seal.get('ordinary_ci_inventory')): + or seal.get('ordinary_ci_inventory') or seal.get('readiness_observation')): if store.state.claim_for(db, spec['work_id']) is not None: - raise ValueError('listener verification admission lost its released claim') + raise ValueError('gate retry admission lost its released claim') for table, key, expected in [('delivery_attempts', 'job_key', seal['attempts']), ('delivery_effects', 'effect_key', seal['effects'])]: records = [dict(item) for item in db.execute( f'SELECT * FROM {table} WHERE run_id=? ORDER BY {key}', (run_id,))] if records != expected: - raise ValueError('listener verification admission lost its full ledger') + raise ValueError('gate retry admission lost its full ledger') if work_binding(store, spec, db) != seal['work_binding']: raise ValueError('gate retry admission lost issue ownership') if ((preflight_generation or seal.get('preparation_observation') - or layout or verification or ordering) + or seal.get('readiness_observation') or layout or verification or ordering) and spec.get('feature_worker')): require_worker_handoff(store, spec) if preparation_owner is not None: @@ -1171,6 +1240,8 @@ def gate_namespace(generation, stage=None, *, preflight_generation=0, prepared_h suffix = f'-{preflight_generation}' if preflight_generation > 1 else '' return gate_namespace(generation, stage) + '/preflight-continuation' + suffix if stage is not None: + if stage == 'readiness' and type(generation) is int and generation in (1, 2): + return f'readiness-gates-admission-{generation}' if stage == 'prerequisite' and type(generation) is int and generation == 1: return 'ordinary-prerequisite-gates-admission' if stage == 'preparation' and type(generation) is int and generation in (1, 2): @@ -1226,6 +1297,17 @@ def readback(store, spec, recovery, *, resource_spec=None): if resource_spec is not None else DeliveryBroker(store, spec)) if effective != spec or broker.candidate() != recovery['candidate']: raise ValueError('gate retry source or execution authority changed') + if expected := recovery['seal'].get('readiness_observation'): + from .delivery_readiness_gate_retry import observe + + seal = recovery['seal'] + _, attempts, effects, claim = _rows(store, spec['run_id']) + closed = store._completed_temporal_result( + spec['run_id'], workflow_id=seal['closed']['workflow_id']) + if (closed != seal['closed'] or attempts != seal['attempts'] or effects != seal['effects'] + or claim is None or claim['owner'] != f"external:devflow:{spec['run_id']}" + or observe(seal['original_spec'], recovery['state']) != expected): + raise ValueError('readiness retry original evidence or ledger changed') if recovery['seal'].get('ordinary_ci_inventory'): from .delivery_ci_inventory import validate diff --git a/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py b/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py new file mode 100644 index 00000000..47cd5ca0 --- /dev/null +++ b/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py @@ -0,0 +1,127 @@ +"""Authenticate an unlaunched gate readiness refusal for bounded reassessment. + +The immutable refusal is historical evidence, not a readiness pass or a launch +permission. The admitted continuation must probe again before any required gate. +""" +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +from .contracts import digest +from .delivery_runtime_location import private_bytes + +RETRY_LIMIT = 2 +SLOTS = {"prepublish": 100, "local": 101, "browser_qa": 102} + + +def probe_attempt(spec, candidate, slot): + """Reserve disjoint readiness evidence names for the two retry generations.""" + if slot not in SLOTS.values(): + raise ValueError("unsupported gate readiness slot") + if spec.get("gate_retry_stage") == "readiness": + generation = spec.get("gate_retry_generation") + if type(generation) is not int or not 1 <= generation <= RETRY_LIMIT: + raise ValueError("readiness retry generation exceeds its finite bound") + # Existing hash-derived attempts are in [10, 900009]. + return 900010 + (generation - 1) * 3 + slot - 100 + generation = int(digest({"spec": spec, "candidate": candidate})[:5], 16) % 300000 + return 10 + generation * 3 + slot - 100 + + +def failure(readiness, stage, candidate_id, cleanup): + state = readiness.get("state") + classification = ("environment_prelaunch" if state == "failed" and cleanup == "confirmed" + else "cancelled" if state == "cancelled" else "unknown") + return {"version": 1, "class": classification, "source": "execution_readiness", + "stage": stage, "candidate_id": candidate_id, + "receipt_sha256": readiness.get("receipt_sha256"), "launched": False} + + +def possible(state): + """Selection hint only. `observe` authenticates the result and its receipt.""" + checks = state.get("checks") or {} + if not isinstance(checks, dict): + return None + found = [stage for stage in SLOTS if isinstance(checks.get(stage), dict) + and checks[stage].get("failure_kind") == "readiness_preparation" + and checks[stage].get("launched") is False] + return found[0] if len(found) == 1 else None + + +def observe(spec, state): + """Bind a completed controller result to its original private failure receipt.""" + stage = possible(state) + checks, candidate = state.get("checks") or {}, state.get("candidate") or {} + roles = state.get("roles") + if (not isinstance(roles, list) or any(not isinstance(role, dict) for role in roles)): + raise ValueError("readiness retry lacks its complete role history") + if (spec.get("readiness_version") != 1 or stage is None + or type(state.get("iteration")) is not int or state["iteration"] < 0 + or not isinstance(candidate, dict) or not candidate.get("id")): + raise ValueError("readiness retry requires an exact unlaunched gate refusal") + expected_error = {"prepublish": "execution readiness prevented prepublication", + "local": "execution readiness prevented local checks", + "browser_qa": "execution readiness prevented browser QA"}[stage] + if state.get("error") != expected_error: + raise ValueError("readiness retry lost its exact controller stop") + result = checks[stage] + receipt = result.get("readiness") + if (not isinstance(receipt, dict) or result.get("state") != "failed" + or result.get("cleanup") != "confirmed" or result.get("source_unchanged") is not True + or result.get("candidate_id") != candidate["id"] + or checks.get("readiness") != receipt or receipt.get("version") != 1 + or receipt.get("stage") != "plan" or receipt.get("state") != "failed" + or receipt.get("iteration") != state["iteration"] + or receipt.get("attempt") != probe_attempt(spec, candidate, SLOTS[stage])): + raise ValueError("readiness retry lacks a confirmed candidate-bound refusal") + provenance = failure(receipt, stage, candidate["id"], "confirmed") + if result.get("failure_class") not in (None, provenance): + raise ValueError("readiness failure provenance changed") + # A failed or unassessed product gate remains authoritative. Never replace + # pending findings with a same-candidate retry. + for name in ("prepublish", "local", "browser_qa", "review", "qa", "ci"): + item = checks.get(name) + if name != stage and isinstance(item, dict) and item.get("state") != "passed": + raise ValueError("readiness retry cannot replace another gate verdict") + if any(role.get("iteration") == state["iteration"] + and role.get("role") in {"review", "verify"} + and (role.get("status") != "pass" or role.get("findings")) + for role in roles): + raise ValueError("readiness retry cannot replace an independent product finding") + pending = state.get("repair_findings") or {} + if not isinstance(pending, dict): + raise ValueError("readiness retry lacks its pending finding history") + if pending.get("iteration") == state["iteration"] and pending.get("findings"): + raise ValueError("readiness retry cannot discard pending product findings") + subject = receipt.get("binding") or {} + observations = receipt.get("observations") + if (not isinstance(subject, dict) or subject.get("version") != 1 + or subject.get("spec_sha256") != digest(spec) or subject.get("candidate") != candidate + or not isinstance(subject.get("requirements_sha256"), str) + or not re.fullmatch(r"[0-9a-f]{64}", subject["requirements_sha256"]) + or type(subject.get("resource_generation")) is not int + or subject["resource_generation"] < 0 or not isinstance(subject.get("resources"), dict) + or not isinstance(observations, dict) or not observations + or not any(isinstance(item, dict) and item.get("state") == "failed" + for item in observations.values()) + or any(not isinstance(item, dict) or not isinstance(item.get("state"), str) + or item.get("state") not in {"passed", "failed"} + or item.get("cleanup") != "confirmed" or item.get("candidate") != candidate + or item.get("requirement_sha256") != subject["requirements_sha256"] + for item in observations.values())): + raise ValueError("readiness retry lacks complete confirmed failure observations") + path = (Path(spec["state_dir"]) / "readiness" / "receipts" / "plan" + / str(state["iteration"]) / f"{receipt['attempt']}.json") + if receipt.get("receipt") != str(path): + raise ValueError("readiness failure receipt left its owned run") + raw = private_bytes(path, maximum=8 * 1024 * 1024) + if (hashlib.sha256(raw).hexdigest() != receipt.get("receipt_sha256") + or json.loads(raw) != {key: value for key, value in receipt.items() + if key not in {"receipt", "receipt_sha256"}}): + raise ValueError("readiness failure receipt changed") + return {"stage": stage, "candidate_id": candidate["id"], "attempt": receipt["attempt"], + "receipt": str(path), "receipt_sha256": receipt["receipt_sha256"], + "requirements_sha256": subject["requirements_sha256"], "failure_class": provenance} diff --git a/runtime/src/devflow_temporal/delivery_workflow.py b/runtime/src/devflow_temporal/delivery_workflow.py index 3ddb186e..80f90ed3 100644 --- a/runtime/src/devflow_temporal/delivery_workflow.py +++ b/runtime/src/devflow_temporal/delivery_workflow.py @@ -2664,6 +2664,16 @@ async def _run_iterations( return await self._stop(spec, 'technical prepublication checks failed: ' + type(exc).__name__, cause=exc) self.state['checks']['prepublish'] = checked + if (spec.get('gate_retry_stage') == 'readiness' + and spec.get('readiness_version') == 1 + and checked.get('readiness')): + self.state['checks']['readiness'] = checked['readiness'] + if (checked.get('failure_kind') == 'readiness_preparation' + and checked.get('launched') is False): + if checked.get('cleanup') == 'unknown': + self.state['cleanup'] = 'unknown' + return await self._stop( + spec, 'execution readiness prevented prepublication') if checked.get('state') != 'passed' or checked.get('cleanup') == 'unknown': return await self._stop(spec, 'technical prepublication checks failed') elif recovery is not None and iteration == start_iteration: diff --git a/runtime/tests/test_delivery_automatic_retry.py b/runtime/tests/test_delivery_automatic_retry.py index 08bf9ff7..b9b67d66 100644 --- a/runtime/tests/test_delivery_automatic_retry.py +++ b/runtime/tests/test_delivery_automatic_retry.py @@ -30,7 +30,7 @@ def stopped(service, monkeypatch): 'closed_at': '2026-10-07T00:00:00+00:00', 'request_digest': spec['request_digest'], 'recovery_digest': None, 'result': {'run_id': request['run_id'], 'outcome': 'blocked', 'phase': 'blocked', 'execution_state': 'blocked', - 'cleanup': 'confirmed', 'checks': checks}} + 'cleanup': 'confirmed', 'checks': checks, 'roles': []}} monkeypatch.setattr(DeliveryStore, '_completed_temporal_result', lambda *_a, **_k: closed) monkeypatch.setattr(retry, 'observe_finalized_resources', lambda *_a, **_k: {'finalization_sha256': 'fixture-finalization'}) @@ -150,3 +150,16 @@ def test_automatic_generations_spend_one_original_total_ceiling(service, monkeyp assert retry.retry_once(store) == [] with store._connect() as db: assert db.execute('SELECT COUNT(*) FROM delivery_runs').fetchone()[0] == 3 + +def test_transient_stop_after_implementation_retains_candidate_instead_of_fresh_run( + service, monkeypatch, +): + store, request, closed = stopped(service, monkeypatch) + closed['result'].update(candidate={'id': 'retained-candidate'}, iteration=0, roles=[{ + 'role': 'implement', 'iteration': 0, 'status': 'pass', 'session_id': 'retained-session'}]) + assert retry.retry_once(store) == [] + with store._connect() as db: + assert db.execute('SELECT COUNT(*) FROM delivery_runs').fetchone()[0] == 1 + events = [json.loads(row[0]) for row in db.execute( + "SELECT payload_json FROM delivery_events WHERE type='automatic_retry_skipped'")] + assert any('same-candidate continuation' in event['reason'] for event in events) diff --git a/runtime/tests/test_delivery_feature_readiness_handoff.py b/runtime/tests/test_delivery_feature_readiness_handoff.py new file mode 100644 index 00000000..37a300aa --- /dev/null +++ b/runtime/tests/test_delivery_feature_readiness_handoff.py @@ -0,0 +1,195 @@ +"""Observe the feature-worker readiness handoff boundary without a provider.""" + +import json +from copy import deepcopy + +import pytest +from test_delivery_feature_execution import feature_service +from test_delivery_store import service as service + +from devflow_temporal import delivery_feature_activities as activities +from devflow_temporal.contracts import canonical_json, digest +from devflow_temporal.delivery_activities import _check_readiness +from devflow_temporal.delivery_broker import DeliveryBroker +from devflow_temporal.delivery_config import DeliveryConfig +from devflow_temporal.delivery_execution_registry import OwnershipConflict +from devflow_temporal.delivery_feature_execution import ( + register_worker, + registry, + worker_key, + worker_spec, +) +from devflow_temporal.delivery_github_contract import ordered_chunks +from devflow_temporal.delivery_resources import RunResources + + +def test_feature_readiness_requires_reserved_successor_and_preserves_budget(service, monkeypatch): + store, request, _ = feature_service(service, monkeypatch) + original_admit = DeliveryConfig.admit + + def admit(config, *args, **kwargs): + spec = original_admit(config, *args, **kwargs) + spec["readiness_version"] = 1 + spec["policy"]["readiness"] = { + "version": 1, + "generation": 0, + "browser_engines": [], + "temporal_checks": [], + } + spec["policy_digest"] = digest(spec["policy"]) + return spec + + monkeypatch.setattr(DeliveryConfig, "admit", admit) + store.submit(request) + parent = store.effective_spec(request["run_id"]) + chunk = ordered_chunks(json.loads(request["accepted_plan"]))[0] + child = worker_spec( + parent, + chunk, + {"url": "https://github.com/example/fixture/issues/10"}, + kind="chunk", + base_sha=parent["base_sha"], + base_branch="main", + ) + register_worker(store, parent, child) + broker = DeliveryBroker(store, child) + broker.prepare() + resources = RunResources(child) + stage = broker.checkout.with_name("staged-feature-candidate") + broker.checkout.rename(stage) + resources.register(broker.checkout, "checkout") + stage.rename(broker.checkout) + resources.created(broker.checkout) + (broker.checkout / "README.md").write_text("Retained feature candidate\n") + candidate = broker.candidate() + monkeypatch.setattr( + "devflow_temporal.delivery_preparation.verify_prepared_spec", lambda *_: None + ) + monkeypatch.setattr( + "devflow_temporal.delivery_configured_resources.implementation_prerequisites", lambda *_: [] + ) + monkeypatch.setattr( + "devflow_temporal.delivery_plan_checks.planned_checks", lambda *_a, **_k: [] + ) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda *_a, **_k: { + "state": "failed", + "results": [{"id": "missing", "passed": False, "cleanup": "confirmed"}], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + result = _check_readiness(broker, {"spec": child, "candidate": candidate, "iteration": 0}, 100) + implementation = { + "role": "implement", + "iteration": 0, + "status": "pass", + "session_id": "original-implementation", + "cleanup": "confirmed", + "candidate": candidate, + } + checks = {"prepublish": result, "readiness": result["readiness"]} + state = { + "run_id": child["run_id"], + "revision": 9, + "iteration": 0, + "phase": "blocked", + "execution_state": "blocked", + "outcome": "blocked", + "cleanup": "confirmed", + "candidate": candidate, + "pull_request": None, + "roles": [implementation], + "checks": checks, + "error": "execution readiness prevented prepublication", + } + store.project( + child["run_id"], + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=state["error"], + outcome="blocked", + cleanup="confirmed", + candidate=candidate, + checks=checks, + pull_request=None, + error=state["error"], + iteration=0, + protocol_revision=9, + ) + shared = registry(parent) + token = parent["feature_delivery"]["owner"] + shared.finish_worker(token, worker_key(child["run_id"], token), {"cleanup": "confirmed"}) + before = deepcopy(shared.budget(token["issue_id"])) + shared.stop(token, "stopped-for-preparation", {"cleanup": "confirmed"}) + with store._connect() as db: + store.state.release_work(db, parent["work_id"], "external:devflow:" + parent["run_id"]) + store.submit( + { + **request, + "run_id": "run-successor", + "command_id": "continue-preparation", + "feature_predecessor": token, + } + ) + successor = store.effective_spec("run-successor") + with store._connect() as db: + db.execute( + "INSERT INTO delivery_attempts(job_key,run_id,role,iteration,candidate_id," + "state,session_id,result_json,cleanup) VALUES " + "('implementation',?,'implement',0,?,'finished',?,?,'confirmed')", + ( + child["run_id"], + candidate["id"], + implementation["session_id"], + canonical_json( + { + k: v + for k, v in implementation.items() + if k not in {"role", "iteration", "candidate"} + } + ), + ), + ) + store.state.release_work(db, child["work_id"], "external:devflow:" + child["run_id"]) + row = dict( + db.execute("SELECT * FROM delivery_runs WHERE run_id=?", (child["run_id"],)).fetchone() + ) + closed = { + "workflow_id": row["workflow_id"], + "execution_run_id": "closed-worker", + "request_digest": child["request_digest"], + "recovery_digest": None, + "result": state, + } + monkeypatch.setattr(store, "_completed_temporal_result", lambda *_a, **_k: closed) + monkeypatch.setattr(DeliveryBroker, "_existing_pr", lambda *_a, **_k: None) + monkeypatch.setattr( + "devflow_temporal.delivery_gate_retry._stopped_cleanup", lambda *_a, **_k: {} + ) + with pytest.raises(OwnershipConflict, match="active execution assignment"): + activities.resume_worker(store, successor, child, row) + assert shared.budget(token["issue_id"]) == before + with store._connect() as db: + assert not db.execute("SELECT 1 FROM delivery_gate_admissions").fetchone() + next_token = successor["feature_delivery"]["owner"] + shared.reserve_worker( + next_token, + worker_key(child["run_id"], next_token), + child["feature_worker"]["workstream_id"], + ) + continued = activities.resume_worker(store, successor, child, row) + assert continued["resumed"] and continued["spec"]["run_id"] == child["run_id"] + assert shared.budget(token["issue_id"]) == before + with store._connect() as db: + recovery = json.loads( + db.execute( + "SELECT recovery_json FROM delivery_runs WHERE run_id=?", (child["run_id"],) + ).fetchone()[0] + ) + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert recovery["command"]["additional_iterations"] == 0 + assert recovery["state"]["checks"]["prepublish"]["readiness"] == result["readiness"] diff --git a/runtime/tests/test_delivery_readiness_gate_retry.py b/runtime/tests/test_delivery_readiness_gate_retry.py new file mode 100644 index 00000000..d56fbbd6 --- /dev/null +++ b/runtime/tests/test_delivery_readiness_gate_retry.py @@ -0,0 +1,739 @@ +"""Public same-candidate retry of a real, retained readiness refusal. + +The Temporal closure, implementation and failed dependency are controlled +fixtures. Readiness receipt creation, Git/SQLite ownership and HTTP admission +are real; no provider or production installation is used. +""" + +from __future__ import annotations + +import json +from contextlib import asynccontextmanager +from copy import deepcopy +from pathlib import Path + +import httpx +import pytest +from test_delivery_gate_retry import unpublished as unpublished +from test_delivery_metadata_recovery import published as published +from test_delivery_store import _git +from test_delivery_store import service as base_service + +from devflow_temporal.contracts import canonical_json, digest +from devflow_temporal.delivery_activities import _check_readiness +from devflow_temporal.delivery_api import create_app +from devflow_temporal.delivery_broker import DeliveryBroker +from devflow_temporal.delivery_config import DeliveryConfig +from devflow_temporal.delivery_gate_retry import KIND, readback +from devflow_temporal.delivery_resources import RunResources +from devflow_temporal.delivery_store import DeliveryStore + + +@pytest.fixture +def service(tmp_path): + store, request = base_service.__wrapped__(tmp_path) + source = Path(store.config.raw["repositories"]["fixture"]["source_path"]) + (source / "test_retry_gate.py").write_text( + "import unittest\n" + "from pathlib import Path\n" + "class Gate(unittest.TestCase):\n" + " def test_candidate(self):\n" + " self.assertIn('Owned unpublished candidate', Path('README.md').read_text())\n" + " def test_absent(self):\n" + " self.assertIn('an absent requirement', Path('README.md').read_text())\n" + ) + _git(source, "add", "test_retry_gate.py") + _git(source, "commit", "-qm", "Add fixture check") + config = deepcopy(store.config.raw) + config["repositories"]["fixture"]["expected_base_sha"] = _git(source, "rev-parse", "HEAD") + store.config.path.write_text(json.dumps(config)) + return DeliveryStore(DeliveryConfig.load(store.config.path)), request + + +@pytest.fixture +def stopped_readiness(unpublished, monkeypatch, request): + store, _, state, command = unpublished + spec = deepcopy(store.spec("run-1")) + spec["readiness_version"] = 1 + spec["policy"]["prepublish_checks"] = [ + { + "id": "diff", + "argv": ["git", "diff", "--check"], + "cwd": ".", + "kind": "check", + "timeout_seconds": 30, + } + ] + selector = ( + "test_retry_gate.Gate.test_absent" + if getattr(request, "param", "") == "assertion" + else "test_retry_gate.Gate.test_candidate" + ) + spec["policy"]["checks"] = [ + { + "id": "assert-readme", + "argv": ["python3", "-m", "unittest", selector], + "cwd": ".", + "kind": "check", + "timeout_seconds": 30, + } + ] + spec["policy"]["readiness"] = { + "version": 1, + "generation": 0, + "browser_engines": [], + "temporal_checks": [], + } + browser_missing = getattr(request, "param", "") == "missing-browser" + if browser_missing: + spec["policy"]["browser_qa"] = {"argv": ["playwright", "test"], "ports": {}} + spec["policy"]["readiness"]["browser_engines"] = ["chromium"] + monkeypatch.setattr(RunResources, "browser_scratch", lambda self: Path(spec["state_dir"])) + monkeypatch.setattr( + "devflow_temporal.delivery_sandbox.prepare_browser_qa", + lambda *_args, **_kwargs: (None, {"PATH": str(Path(spec["state_dir"]) / "no-node")}), + ) + spec["policy_digest"] = digest(spec["policy"]) + with store._connect() as db: + db.execute( + "UPDATE delivery_runs SET request_json=? WHERE run_id=?", + (canonical_json(spec), "run-1"), + ) + broker = DeliveryBroker(store, spec) + resources = RunResources(spec) + staged = broker.checkout.with_name("staged-candidate") + broker.checkout.rename(staged) + resources.register(broker.checkout, "checkout") + staged.rename(broker.checkout) + resources.created(broker.checkout) + candidate = broker.candidate() + state["candidate"] = candidate + state["roles"][0]["candidate"] = candidate + with store._connect() as db: + role_result = { + k: v + for k, v in state["roles"][0].items() + if k not in {"role", "iteration", "candidate"} + } + db.execute( + "UPDATE delivery_attempts SET result_json=? WHERE job_key='original'", + (canonical_json(role_result),), + ) + command["expected_candidate_id"] = candidate["id"] + monkeypatch.setattr( + "devflow_temporal.delivery_preparation.verify_prepared_spec", lambda *_: None + ) + monkeypatch.setattr( + "devflow_temporal.delivery_configured_resources.implementation_prerequisites", lambda *_: [] + ) + monkeypatch.setattr( + "devflow_temporal.delivery_plan_checks.planned_checks", lambda *_a, **_k: [] + ) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, *_a, **_k: { + "state": "failed", + "results": [{"id": "dependency", "passed": False, "cleanup": "confirmed"}], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + + def failure(execution, current): + broker = DeliveryBroker(store, execution) + result = _check_readiness( + broker, {"spec": execution, "candidate": current["candidate"], "iteration": 0}, 100 + ) + assert result["failure_class"]["class"] == "environment_prelaunch" + assert result["readiness"]["state"] == "failed" + current["checks"] = {"prepublish": result, "readiness": result["readiness"]} + current["error"] = "execution readiness prevented prepublication" + return result + + original = failure(spec, state) + store.project( + "run-1", + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=state["error"], + candidate=candidate, + checks=state["checks"], + iteration=0, + protocol_revision=9, + outcome="blocked", + cleanup="confirmed", + error=state["error"], + ) + closed = { + "workflow_id": "delivery-run-1", + "execution_run_id": "closed-original", + "request_digest": spec["request_digest"], + "recovery_digest": None, + "result": state, + } + monkeypatch.setattr(store, "_completed_temporal_result", lambda *_a, **_k: closed) + monkeypatch.setattr(DeliveryStore, "_completed_temporal_result", lambda *_a, **_k: closed) + return store, broker, state, command, original, closed, failure + + +@asynccontextmanager +async def client_for(store): + app = create_app(store.config.path) + host = store.config.dashboard_url + client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=app, client=("127.0.0.1", 8001)), base_url=host + ) + async with client: + headers = { + "Origin": host, + "X-Devflow-CSRF": (await client.get("/api/session")).json()["csrf_token"], + } + yield client, headers + + +@pytest.mark.asyncio +async def test_public_retry_preserves_receipts_and_bounds_repeated_failures(stopped_readiness): + store, broker, state, command, original, closed, failure = stopped_readiness + candidate = deepcopy(state["candidate"]) + retained = Path(original["readiness"]["receipt"]).read_bytes() + with store._connect() as db: + initial_attempts = [dict(x) for x in db.execute("SELECT * FROM delivery_attempts")] + initial_effects = [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] + attempts = [original["readiness"]["attempt"]] + async with client_for(store) as (client, headers): + for generation in (1, 2): + preflight = await client.post( + "/api/runs/run-1/repair-admission-preflight", json=command + ) + assert preflight.status_code == 200, preflight.text + result = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert result.status_code == 200, result.text + admitted = result.json() + assert admitted["implementation_authority"] is False + assert ( + admitted["additional_iterations"] == 0 + and admitted["candidate_id"] == candidate["id"] + ) + assert ( + await client.post("/api/runs/run-1/continue-repair", json=command, headers=headers) + ).json() == admitted + execution = store.effective_spec("run-1") + with store._connect() as db: + recovery = json.loads( + db.execute( + "SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'" + ).fetchone()[0] + ) + assert [ + dict(x) for x in db.execute("SELECT * FROM delivery_attempts") + ] == initial_attempts + assert [ + dict(x) for x in db.execute("SELECT * FROM delivery_effects") + ] == initial_effects + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert execution["gate_retry_generation"] == generation + assert execution["gate_retry_stage"] == "readiness" + assert recovery["state"]["checks"]["prepublish"]["readiness"]["state"] == "failed" + assert readback(store, execution, recovery) is None + if generation == 1: + old_path = Path(original["readiness"]["receipt"]) + old_path.write_bytes(retained + b" ") + with pytest.raises(ValueError, match="receipt changed"): + readback(store, execution, recovery) + old_path.write_bytes(retained) + assert broker.candidate() == candidate + current = deepcopy(recovery["state"]) + current.update( + revision=9 + generation, + phase="blocked", + execution_state="blocked", + outcome="blocked", + cleanup="confirmed", + candidate=recovery["candidate"], + ) + failed_again = failure(execution, current) + attempts.append(failed_again["readiness"]["attempt"]) + store.project( + "run-1", + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=current["error"], + candidate=current["candidate"], + checks=current["checks"], + iteration=0, + protocol_revision=current["revision"], + outcome="blocked", + cleanup="confirmed", + error=current["error"], + ) + with store._connect() as db: + store.state.release_work(db, execution["work_id"], "external:devflow:run-1") + closed.update( + workflow_id=admitted["workflow_id"], + execution_run_id=f"closed-{generation}", + recovery_digest=digest(recovery), + result=current, + ) + command = { + **command, + "command_id": f"retry-readiness-{generation + 1}", + "expected_revision": current["revision"], + } + refusal = await client.post("/api/runs/run-1/repair-admission-preflight", json=command) + assert refusal.status_code == 409 + assert "readiness_retry_limit_exhausted" in refusal.text + assert len(set(attempts)) == 3 + assert Path(original["readiness"]["receipt"]).read_bytes() == retained + with store._connect() as db: + assert [dict(x) for x in db.execute("SELECT * FROM delivery_attempts")] == initial_attempts + assert [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] == initial_effects + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drift", ["receipt", "source", "effect", "product", "cleanup"]) +async def test_public_retry_refuses_changed_or_unsettled_evidence(stopped_readiness, drift): + store, broker, state, command, original, _, _ = stopped_readiness + if drift == "receipt": + path = Path(original["readiness"]["receipt"]) + path.write_bytes(path.read_bytes() + b" ") + elif drift == "source": + (broker.checkout / "README.md").write_text("Changed candidate\n") + elif drift == "effect": + with store._connect() as db: + db.execute( + "INSERT INTO delivery_effects VALUES ('pending','run-1','publish'," + "'{}','pending',NULL,'now')" + ) + elif drift == "product": + state["roles"].append( + { + "role": "review", + "iteration": 0, + "status": "findings", + "findings": ["Real product defect"], + "cleanup": "confirmed", + } + ) + else: + state["cleanup"] = "unknown" + with store._connect() as db: + db.execute("UPDATE delivery_runs SET cleanup='unknown' WHERE run_id='run-1'") + async with client_for(store) as (client, _): + response = await client.post("/api/runs/run-1/repair-admission-preflight", json=command) + assert response.status_code == 409, response.text + with store._connect() as db: + assert not db.execute("SELECT 1 FROM delivery_gate_admissions").fetchone() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stopped_readiness", ["passing", "assertion"], indirect=True) +async def test_retry_workflow_runs_the_frozen_gates_without_implementation( + stopped_readiness, monkeypatch +): + import asyncio + + from temporal_test_server import local_temporal + from temporalio import activity + from temporalio.worker import Worker + + from devflow_temporal.delivery_activities import ( + delivery_checks, + delivery_gates_readback, + delivery_precheck, + delivery_project, + ) + from devflow_temporal.delivery_workflow import DeliveryWorkflow + + store, broker, state, command, original, _, _ = stopped_readiness + old_receipt = Path(original["readiness"]["receipt"]).read_bytes() + async with client_for(store) as (client, headers): + response = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert response.status_code == 200, response.text + admitted = response.json() + spec = store.effective_spec("run-1") + with store._connect() as db: + recovery = json.loads( + db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0] + ) + store.mark_start("run-1", accepted=True) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, _i, candidate, **_k: { + "state": "passed", + "results": [], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + monkeypatch.setattr( + "devflow_temporal.delivery_readiness.probe_tools", + lambda _broker, requirements, candidate, **_kw: { + "state": "passed", + "cleanup": "confirmed", + "component": "tools", + "candidate": candidate, + "requirement_sha256": digest(requirements), + "items": [], + }, + ) + monkeypatch.setattr("devflow_temporal.delivery_readiness_verify.tools", lambda *_a: None) + actual_gate_checkout = DeliveryBroker.gate_checkout + + def gate_checkout(self, role, iteration, candidate): + path = actual_gate_checkout(self, role, iteration, candidate) + resources = self.run_resources() + with resources.locked(read_only=True) as manifest: + registered = str(path) in manifest["roots"] + if not registered: + staged = path.with_name(path.name + "-staged") + path.rename(staged) + resources.register(path, "gate") + staged.rename(path) + resources.created(path, head=candidate["head"]) + return path + + monkeypatch.setattr(DeliveryBroker, "gate_checkout", gate_checkout) + calls = [] + + @activity.defn(name="delivery_publish") + async def publish(body): + calls.append("publish") + assert body["candidate"]["id"] == state["candidate"]["id"] + _git(broker.checkout, "add", "README.md") + _git(broker.checkout, "commit", "-qm", "Publish fixture") + current = DeliveryBroker(store, body["spec"]).candidate() + return { + "number": 7, + "url": "https://example.invalid/pull/7", + "state": "OPEN", + "head": current["head"], + "candidate": current, + } + + @activity.defn(name="delivery_role") + async def role(body): + calls.append(body["role"]) + assert body["role"] in {"review", "verify"} + return { + "role": body["role"], + "iteration": body["iteration"], + "status": "pass", + "session_id": "independent-" + body["role"], + "candidate": body["candidate"], + "cleanup": "confirmed", + "findings": [], + } + + @activity.defn(name="delivery_tracker_start") + async def start(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_tracker") + async def tracker(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_ci") + async def ci(_body): + calls.append("ci") + return {"state": "passed"} + + async with local_temporal() as env: + async with Worker( + env.client, + task_queue="issue-243-gates", + workflows=[DeliveryWorkflow], + activities=[ + delivery_gates_readback, + delivery_project, + delivery_precheck, + delivery_checks, + publish, + role, + start, + tracker, + ci, + ], + ): + handle = await env.client.start_workflow( + DeliveryWorkflow.run, + args=[spec, recovery], + id=admitted["workflow_id"], + task_queue="issue-243-gates", + ) + result = await asyncio.wait_for(handle.result(), 60) + if spec["policy"]["checks"][0]["argv"][-1].endswith("test_absent"): + assert result["outcome"] == "blocked" + assert result["checks"]["local"]["state"] == "failed" + assert result["checks"]["local"]["results"][0]["exit_code"] != 0 + assert any("local_checks" in finding for finding in result["findings"]) + assert calls == ["publish"] + else: + assert result["outcome"] == "delivered", result + assert calls == ["publish", "review", "verify", "ci"] + assert result["iteration"] == 0 + assert result["roles"][0] == state["roles"][0] + assert ( + result["checks"]["prepublish"]["results"][0]["argv"] + == spec["policy"]["prepublish_checks"][0]["argv"] + ) + assert result["checks"]["local"]["results"][0]["argv"] == spec["policy"]["checks"][0]["argv"] + assert recovery["original_spec"]["policy"]["checks"] == spec["policy"]["checks"] + assert Path(original["readiness"]["receipt"]).read_bytes() == old_receipt + with store._connect() as db: + assert db.execute("SELECT COUNT(*) FROM delivery_attempts").fetchone()[0] == 1 + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + + +@pytest.mark.asyncio +async def test_public_native_retry_reports_missing_descendant_settlement(stopped_readiness): + """A plausible native-shaped checkpoint still cannot bypass missing settlement proof.""" + import hashlib + + from devflow_temporal.delivery_readiness_gate_retry import failure as classify + from devflow_temporal.delivery_readiness_gate_retry import probe_attempt + from devflow_temporal.delivery_resources import write_private + + store, _, state, command, result, closed, _ = stopped_readiness + spec = deepcopy(store.spec("run-1")) + spec["provider"] = "codex" + spec["policy"]["host_sandbox"] = "trusted-local" + spec["policy"]["execution_backend"] = "native-macos" + spec["policy_digest"] = digest(spec["policy"]) + candidate = deepcopy(state["candidate"]) + candidate["policy_digest"] = spec["policy_digest"] + state["candidate"] = candidate + state["roles"][0]["candidate"] = candidate + receipt = result["readiness"] + receipt["binding"]["spec_sha256"] = digest(spec) + receipt["binding"]["candidate"] = candidate + for observation in receipt["observations"].values(): + observation["candidate"] = candidate + receipt["attempt"] = probe_attempt(spec, candidate, 100) + path = ( + Path(spec["state_dir"]) + / "readiness" + / "receipts" + / "plan" + / "0" + / f"{receipt['attempt']}.json" + ) + receipt["receipt"] = str(path) + write_private( + path, + {key: value for key, value in receipt.items() if key not in {"receipt", "receipt_sha256"}}, + ) + receipt["receipt_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + result["candidate_id"] = candidate["id"] + result["failure_class"] = classify(receipt, "prepublish", candidate["id"], "confirmed") + with store._connect() as db: + role_result = { + k: v + for k, v in state["roles"][0].items() + if k not in {"role", "iteration", "candidate"} + } + db.execute( + "UPDATE delivery_attempts SET result_json=? WHERE job_key='original'", + (canonical_json(role_result),), + ) + db.execute( + "UPDATE delivery_runs SET request_json=?,candidate_json=?,checks_json=? " + "WHERE run_id='run-1'", + (canonical_json(spec), canonical_json(candidate), canonical_json(state["checks"])), + ) + closed["result"] = state + async with client_for(store) as (client, headers): + assessment = await client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert assessment.status_code == 200, assessment.text + data = assessment.json() + assert data["eligible"] is False + assert any( + item["code"] == "native_descendant_settlement_unknown" and item["satisfied"] is None + for item in data["blockers"] + ), data + denied = await client.post("/api/runs/run-1/continue-repair", json=command, headers=headers) + assert denied.status_code == 409, denied.text + assert any( + item["code"] == "native_descendant_settlement_unknown" + for item in denied.json()["detail"]["blockers"] + ) + with store._connect() as db: + assert not db.execute("SELECT 1 FROM delivery_gate_admissions").fetchone() + + +@pytest.mark.asyncio +async def test_public_published_readiness_retains_candidate_and_full_selector( + published, monkeypatch +): + store, _, state, closed, _, _ = published + spec = deepcopy(store.spec("run-1")) + spec["readiness_version"] = 1 + spec["resource_cleanup_version"] = 1 + spec["policy"]["readiness"] = { + "version": 1, + "generation": 0, + "browser_engines": [], + "temporal_checks": [], + } + spec["policy_digest"] = digest(spec["policy"]) + with store._connect() as db: + db.execute( + "UPDATE delivery_runs SET request_json=? WHERE run_id=?", + (canonical_json(spec), "run-1"), + ) + broker = DeliveryBroker(store, spec) + resources = RunResources(spec) + staged = broker.checkout.with_name("staged-published-candidate") + broker.checkout.rename(staged) + resources.register(broker.checkout, "checkout") + staged.rename(broker.checkout) + resources.created(broker.checkout) + candidate = broker.candidate() + state["candidate"] = candidate + state["pull_request"]["candidate"] = candidate + state["roles"][-1]["candidate"]["policy_digest"] = spec["policy_digest"] + with store._connect() as db: + role_result = { + k: v + for k, v in state["roles"][-1].items() + if k not in {"role", "iteration", "candidate", "input_candidate_id"} + } + db.execute( + "UPDATE delivery_attempts SET result_json=? WHERE job_key='implementation-1'", + (canonical_json(role_result),), + ) + store.state.release_work(db, spec["work_id"], "external:devflow:run-1") + monkeypatch.setattr( + "devflow_temporal.delivery_preparation.verify_prepared_spec", lambda *_: None + ) + monkeypatch.setattr( + "devflow_temporal.delivery_configured_resources.implementation_prerequisites", lambda *_: [] + ) + monkeypatch.setattr( + "devflow_temporal.delivery_plan_checks.planned_checks", lambda *_a, **_k: [] + ) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, *_a, **_k: { + "state": "failed", + "results": [{"id": "dependency", "passed": False, "cleanup": "confirmed"}], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + readiness = _check_readiness( + broker, {"spec": spec, "candidate": candidate, "iteration": 1}, 101 + ) + assert readiness["failure_class"]["class"] == "environment_prelaunch" + checks = { + "prepublish": { + "state": "passed", + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + "local": readiness, + "readiness": readiness["readiness"], + } + state.update( + checks=checks, cleanup="confirmed", error="execution readiness prevented local checks" + ) + store.project( + "run-1", + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=state["error"], + candidate=candidate, + pull_request=state["pull_request"], + checks=checks, + iteration=1, + protocol_revision=13, + outcome="blocked", + cleanup="confirmed", + error=state["error"], + ) + closed["result"] = state + monkeypatch.setattr(DeliveryStore, "_completed_temporal_result", lambda *_a, **_k: closed) + monkeypatch.setattr("devflow_temporal.delivery_gate_retry._stopped_cleanup", lambda *a, **k: {}) + monkeypatch.setattr("devflow_temporal.delivery_gate_retry.preserve_resources", lambda *_: None) + command = { + "continuation_kind": KIND, + "command_id": "published-readiness-retry", + "expected_revision": 13, + "expected_iteration": 1, + "expected_candidate_id": candidate["id"], + "expected_pr_number": 7, + "expected_pr_head": candidate["head"], + "additional_iterations": 0, + } + with store._connect() as db: + effects = [ + dict(row) for row in db.execute("SELECT * FROM delivery_effects ORDER BY effect_key") + ] + attempts = [ + dict(row) for row in db.execute("SELECT * FROM delivery_attempts ORDER BY job_key") + ] + async with client_for(store) as (client, headers): + narrower = await client.post( + "/api/runs/run-1/repair-admission-preflight", + json={**command, "verification_test_paths": ["test_retry_gate.py"]}, + ) + assert narrower.status_code == 409 and "frozen test selectors" in narrower.text + preflight = await client.post("/api/runs/run-1/repair-admission-preflight", json=command) + assert preflight.status_code == 200, preflight.text + response = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert response.status_code == 200, response.text + assert response.json()["implementation_authority"] is False + execution = store.effective_spec("run-1") + with store._connect() as db: + recovery = json.loads( + db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0] + ) + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_effects ORDER BY effect_key") + ] == effects + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_attempts ORDER BY job_key") + ] == attempts + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert recovery["publication"] == state["pull_request"] + assert recovery["state"]["checks"]["local"] == readiness + assert execution["policy"]["checks"] == spec["policy"]["checks"] + assert readback(store, execution, recovery) == state["pull_request"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stopped_readiness", ["missing-browser"], indirect=True) +async def test_public_retry_retains_missing_browser_runtime_observation(stopped_readiness): + store, _, state, command, original, _, _ = stopped_readiness + observation = original["readiness"]["observations"]["browser"] + assert observation["state"] == "failed" and observation["launched"] is False + assert observation["reason"] == "node unavailable in browser environment" + async with client_for(store) as (client, headers): + response = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert response.status_code == 200, response.text + assert response.json()["implementation_authority"] is False + with store._connect() as db: + recovery = json.loads( + db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0] + ) + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert db.execute("SELECT COUNT(*) FROM delivery_attempts").fetchone()[0] == 1 + assert recovery["state"]["checks"]["prepublish"]["readiness"] == original["readiness"] + assert recovery["candidate"] == state["candidate"] From c7bd401b51a5c2f3a7a07cfc5ad367e1d15c4489 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:22:14 +0100 Subject: [PATCH 2/8] test(recovery): exercise published candidate retry --- .../test_delivery_readiness_gate_retry.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/runtime/tests/test_delivery_readiness_gate_retry.py b/runtime/tests/test_delivery_readiness_gate_retry.py index d56fbbd6..83c12b40 100644 --- a/runtime/tests/test_delivery_readiness_gate_retry.py +++ b/runtime/tests/test_delivery_readiness_gate_retry.py @@ -715,6 +715,117 @@ async def test_public_published_readiness_retains_candidate_and_full_selector( assert execution["policy"]["checks"] == spec["policy"]["checks"] assert readback(store, execution, recovery) == state["pull_request"] + import asyncio + + from temporal_test_server import local_temporal + from temporalio import activity + from temporalio.worker import Worker + + from devflow_temporal.delivery_activities import ( + delivery_checks, + delivery_finalize_resources, + delivery_gates_readback, + delivery_precheck, + delivery_project, + ) + from devflow_temporal.delivery_workflow import DeliveryWorkflow + + store.mark_start("run-1", accepted=True) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, _i, value, **_k: { + "state": "passed", + "results": [], + "candidate_id": value["id"], + "source_unchanged": True, + }, + ) + monkeypatch.setattr( + "devflow_temporal.delivery_readiness.probe_tools", + lambda _broker, requirements, value, **_kw: { + "state": "passed", + "cleanup": "confirmed", + "component": "tools", + "candidate": value, + "requirement_sha256": digest(requirements), + "items": [], + }, + ) + monkeypatch.setattr("devflow_temporal.delivery_readiness_verify.tools", lambda *_a: None) + calls = [] + + @activity.defn(name="delivery_publish") + async def unexpected_publish(_body): + pytest.fail("published retry must never publish the candidate again") + + @activity.defn(name="delivery_role") + async def role(body): + calls.append(body["role"]) + assert body["role"] in {"review", "verify"} + assert body["candidate"] == candidate + return { + "role": body["role"], + "iteration": body["iteration"], + "status": "pass", + "session_id": "independent-" + body["role"], + "candidate": body["candidate"], + "cleanup": "confirmed", + "findings": [], + } + + @activity.defn(name="delivery_tracker_start") + async def start(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_tracker") + async def tracker(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_ci") + async def ci(body): + calls.append("ci") + assert body["pull_request"] == state["pull_request"] + return {"state": "passed"} + + async with local_temporal() as env: + async with Worker( + env.client, + task_queue="issue-243-published", + workflows=[DeliveryWorkflow], + activities=[ + delivery_gates_readback, + delivery_project, + delivery_precheck, + delivery_checks, + delivery_finalize_resources, + unexpected_publish, + role, + start, + tracker, + ci, + ], + ): + handle = await env.client.start_workflow( + DeliveryWorkflow.run, + args=[execution, recovery], + id=response.json()["workflow_id"], + task_queue="issue-243-published", + ) + finished = await asyncio.wait_for(handle.result(), 60) + assert finished["outcome"] == "delivered", finished + assert calls == ["review", "verify", "ci"] + assert finished["candidate"] == candidate + assert finished["checks"]["local"]["results"][0]["argv"] == spec["policy"]["checks"][0]["argv"] + with store._connect() as db: + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_effects ORDER BY effect_key") + ] == effects + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_attempts ORDER BY job_key") + ] == attempts + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + @pytest.mark.asyncio @pytest.mark.parametrize("stopped_readiness", ["missing-browser"], indirect=True) From c31d6d8fc7e724fab4aab9d168ee292fc93779e6 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:32:56 +0100 Subject: [PATCH 3/8] fix(recovery): authenticate early unreviewed revisions --- docs/trusted-local-delivery.md | 9 + .../delivery_feature_revisions.py | 35 ++++ .../delivery_prelaunch_cleanup.py | 60 ++++--- .../tests/test_delivery_feature_revisions.py | 158 +++++++++++++++--- 4 files changed, 222 insertions(+), 40 deletions(-) diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index 6616d626..0bf6afc4 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -199,6 +199,15 @@ readiness path also requires an already authenticated successor assignment. The current coordinator examines the stopped worker before reserving that assignment, so an unreserved continuation remains blocked rather than bypassing ownership. +For a feature-plan correction stopped before its review launch envelope is +written, the unreviewed-predecessor check can reconstruct the review identity +from the sealed revision, proposal, candidate and owner. It also requires the +supervisor's completed prelaunch result, registered workspace, an empty attempt +directory and the original unheld owner lease. A missing request alone does not +prove the exception, and a real adverse review remains a rejection. Generic +completed-resource observation still requires its existing request evidence for +this early shape. + An older run stopped by that browser-output registration defect can use one `published_check_prelaunch_retry` with zero additional implementation turns. Admission requires its exact closed UNKNOWN checkpoint, passed local checks and diff --git a/runtime/src/devflow_temporal/delivery_feature_revisions.py b/runtime/src/devflow_temporal/delivery_feature_revisions.py index ebbc87e6..c2de5011 100644 --- a/runtime/src/devflow_temporal/delivery_feature_revisions.py +++ b/runtime/src/devflow_temporal/delivery_feature_revisions.py @@ -2261,6 +2261,41 @@ def _unreviewed_rejection(store, spec, context, rejected, values): str(Path(historical["state_dir"]) / "gates" / "0" / "review")), ) request_path = folder / "request.json" + if role == "review" and not os.path.lexists(request_path): + # The revision and proposal identity were reconstructed from + # immutable coordinator checkpoints above, not from a caller. + # A supervisor may refuse preparation before it persists the + # later native launch envelope, so prove the typed terminal + # attempt under its original owner lease instead. + if not attempt: + return None + from .delivery_prelaunch_cleanup import completed_prelaunch + from .delivery_resources import RunResources + + intent = {key: expected[key] for key in ( + "spec", "role", "iteration", "candidate", "resume_session", "revision_context", + "workspace", + )} + manifest = read_private(RunResources(historical).manifest) + with completed_prelaunch(historical, dict(attempt), + {Path(historical["state_dir"]): historical}, manifest, + early_revision_intent=intent) as observed: + with store._connect() as db: + current_attempt = db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", + (owner["run_id"], key), + ).fetchone() + if (not current_attempt or dict(current_attempt) != dict(attempt) + or datetime.fromisoformat(attempt["finished_at"]) > closed_at): + return None + result = json.loads(attempt["result_json"]) + proof = { + "revision_id": revision_id, "review_job_key": key, + "review_intent_digest": observed["revision_intent_sha256"], + "review_result_digest": digest(result), "predecessor_closure": closure, + "prelaunch_evidence": observed, + } + continue if request_path.resolve(strict=True) != request_path: return None request = read_private(request_path) diff --git a/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py b/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py index 8d92dbe9..08ff900d 100644 --- a/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py +++ b/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py @@ -24,12 +24,15 @@ def _stamp(info): @contextmanager -def completed_prelaunch(spec, attempt, executions, manifest, *, completed_intakes=()): +def completed_prelaunch(spec, attempt, executions, manifest, *, completed_intakes=(), + early_revision_intent=None): """Authenticate the existing supervisor's typed pre-monitor failure, never settle it. The durable result is server-owned producer evidence. Neither a missing journal nor a null session suffices. Existing owner/lifecycle leases exclude an unfinished launch; this observer never creates a missing proof or lock. + An early revision identity must be reconstructed by the owning caller from + the sealed predecessor and proposal before it is passed here. """ from .supervisor import DeliverySupervisor @@ -103,20 +106,36 @@ def lease(path, parent): else: locks[name] = lease(folder / name, folder_fd) - fd = os.open('request.json', os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, - dir_fd=folder_fd) - descriptors.append(fd) - info = os.fstat(fd) - if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() - or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 - or info.st_size > _LIMIT): - raise ValueError('completed prelaunch request is not bounded private evidence') - pinned.append((folder / 'request.json', fd, _stamp(info))) - with os.fdopen(os.dup(fd), 'rb') as stream: - raw = stream.read(_LIMIT + 1) - if len(raw) != info.st_size: - raise ValueError('completed prelaunch request changed during observation') - request = json.loads(raw) + early = False + try: + fd = os.open('request.json', os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, + dir_fd=folder_fd) + except FileNotFoundError: + # Only the revision owner may supply a reconstructed identity after + # authenticating its sealed predecessor, proposal and owner records. + # The exact typed supervisor result and the held owner lease above + # establish no launch; an absent request is never sufficient. + if early_revision_intent is None or os.listdir(folder_fd): + raise + early = True + request = early_revision_intent + raw = None + if (not isinstance(request, dict) or request.get('role') != 'review' + or not isinstance(request.get('revision_context'), dict)): + raise ValueError('early prelaunch requires a sealed review identity') from None + else: + descriptors.append(fd) + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() + or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 + or info.st_size > _LIMIT): + raise ValueError('completed prelaunch request is not bounded private evidence') + pinned.append((folder / 'request.json', fd, _stamp(info))) + with os.fdopen(os.dup(fd), 'rb') as stream: + raw = stream.read(_LIMIT + 1) + if len(raw) != info.st_size: + raise ValueError('completed prelaunch request changed during observation') + request = json.loads(raw) if (not isinstance(request, dict) or not isinstance(request.get('spec'), dict) or not isinstance(request.get('candidate'), dict) or not isinstance(request['candidate'].get('id'), str) @@ -136,13 +155,13 @@ def lease(path, parent): if (not admitted or execution.get('provider') != 'codex' or execution.get('policy', {}).get('execution_backend') != 'native-macos' or execution.get('policy_digest') != digest(execution['policy']) - or request.get('native_authorized') is not True + or (not early and request.get('native_authorized') is not True) or DeliverySupervisor._job_key(request) != key or request.get('role') != attempt['role'] or request.get('iteration') != attempt['iteration'] or request['candidate']['id'] != attempt['candidate_id'] - or request.get('result_path') != attempt['result_path'] - or request.get('start_path') != str(folder / 'start.json') + or (not early and (request.get('result_path') != attempt['result_path'] + or request.get('start_path') != str(folder / 'start.json'))) or role not in execution['policy']['roles'] or type(iteration) is not int or iteration < 0 or workspace not in expected_workspaces @@ -154,6 +173,8 @@ def lease(path, parent): raise ValueError('completed prelaunch lost its original admitted request') def unchanged(): + if early and os.listdir(folder_fd): + raise ValueError('early prelaunch attempt is no longer empty') for path, descriptor, stamp in pinned: current, named = os.fstat(descriptor), path.lstat() if any(getattr(current, k) != v or getattr(named, k) != v @@ -172,7 +193,8 @@ def unchanged(): unchanged() yield {'kind': 'completed_native_prelaunch', 'version': 1, 'attempt_sha256': digest(attempt), 'result_sha256': digest(result), - 'request_sha256': hashlib.sha256(raw).hexdigest(), 'leases': locks} + **({'revision_intent_sha256': digest(request)} if early else + {'request_sha256': hashlib.sha256(raw).hexdigest()}), 'leases': locks} unchanged() finally: for fd in reversed(descriptors): diff --git a/runtime/tests/test_delivery_feature_revisions.py b/runtime/tests/test_delivery_feature_revisions.py index 04ba5eaf..610e5791 100644 --- a/runtime/tests/test_delivery_feature_revisions.py +++ b/runtime/tests/test_delivery_feature_revisions.py @@ -2,8 +2,10 @@ from __future__ import annotations +import asyncio import hashlib import json +import os import sys from copy import deepcopy from pathlib import Path @@ -157,7 +159,7 @@ def _revision_diagnostic_for(spec, diagnostic, candidate): def _native_rejected_fixture(native_configuration, monkeypatch, *, prior_cycles=4, - retain_worker=False): + retain_worker=False, early_prelaunch=False): config, request = native_configuration config.raw["max_repairs"] = 10 config.path.write_text(json.dumps(config.raw)) @@ -313,27 +315,49 @@ def observed_build(_store, run_id): **revisions.authenticate_revision_role(store, review), "proposed_plan": value, } - key, _ = supervisor._claim(review) - folder = Path(spec["state_dir"]) / "attempts" / key - review.update( - findings=[], - native_authorized=True, - role_evidence_key=key, - result_path=str(folder / "result.json"), - start_path=str(folder / "start.json"), - workspace=str(Path(spec["state_dir"]) / "gates" / "0" / "review"), - ) - review["review_diff"] = revision_diff(broker, review) - review["review_diff"].pop("candidate_id", None) - write_private(folder / "request.json", review) - with store._connect() as db: - expected = dict( - db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", (key,)).fetchone() + if early_prelaunch: + workspace = broker.gate_checkout("review", 0, candidate) + review["workspace"] = str(workspace) + review["review_diff"] = revision_diff(broker, review) + key = supervisor._job_key(review) + folder = Path(spec["state_dir"]) / "attempts" / key + + def refuse_preparation(*_args, **_kwargs): + raise ValueError("controlled verifier refusal before request persistence") + + def no_process(*_args, **_kwargs): + raise AssertionError("provider was unexpectedly allocated") + + with monkeypatch.context() as refusal: + refusal.setattr("devflow_temporal.delivery_preparation.verify_prepared_spec", + refuse_preparation) + refusal.setattr("devflow_temporal.delivery_native_process.NativeProcess", no_process) + failure = asyncio.run(supervisor.run(review)) + assert failure["finish_reason"] == "prelaunch" and failure["cleanup"] == "confirmed" + assert list(folder.iterdir()) == [] + else: + key, _ = supervisor._claim(review) + folder = Path(spec["state_dir"]) / "attempts" / key + review.update( + findings=[], + native_authorized=True, + role_evidence_key=key, + result_path=str(folder / "result.json"), + start_path=str(folder / "start.json"), + workspace=str(Path(spec["state_dir"]) / "gates" / "0" / "review"), ) - failure = supervisor._native_failure( - review, key, expected, "product patch shape rejected plan comparison", launch_absent=True - ) - assert failure["finish_reason"] == "prelaunch" and failure["cleanup"] == "confirmed" + review["review_diff"] = revision_diff(broker, review) + review["review_diff"].pop("candidate_id", None) + write_private(folder / "request.json", review) + with store._connect() as db: + expected = dict( + db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", (key,)).fetchone() + ) + failure = supervisor._native_failure( + review, key, expected, "product patch shape rejected plan comparison", + launch_absent=True, + ) + assert failure["finish_reason"] == "prelaunch" and failure["cleanup"] == "confirmed" rejection = revisions.reject_revision(store, spec, context["revision_id"], failure["summary"]) row = _stop_revision_fixture(store, spec, shared) return store, spec, shared, value, diagnostic, candidate, context, proposal, rejection, row @@ -578,3 +602,95 @@ def non_gate_drift(*args, **kwargs): revised_worker_spec) response = revisions.revise_feature_plan(store, original["run_id"], command) assert response["repair_budget"]["used"] == 6 + + +@pytest.mark.parametrize("early_prelaunch", [False, True]) +def test_public_revision_retains_unreviewed_prelaunch(native_configuration, monkeypatch, + early_prelaunch): + """A no-launch refusal is not a substantive rejection of a later correction.""" + import httpx + + from devflow_temporal.delivery_api import create_app + + native_configuration[0].raw["execution_mode"] = "trusted-local" + fixture = _native_rejected_fixture(native_configuration, monkeypatch, + early_prelaunch=early_prelaunch) + store, original, shared, plan, diagnostic, _, previous, _, _, row = fixture + command = {"command_id": "prelaunch-revision", "expected_revision": row["revision"], + "reason": "Retry the correction after the environment refusal."} + + async def admit(): + app = create_app(store.config.path) + app.state.delivery.store = store + origin = store.config.dashboard_url.rstrip("/") + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), + base_url=origin) as client: + csrf = (await client.get("/api/session")).json()["csrf_token"] + headers = {"Origin": origin, "X-Devflow-CSRF": csrf} + url = f"/api/runs/{original['run_id']}/revise-feature-plan" + first = await client.post(url, json=command, headers=headers) + replay = await client.post(url, json=command, headers=headers) + assert first.status_code == 200, first.text + assert replay.status_code == 200 and replay.json() == first.json() + return first.json() + + admitted = asyncio.run(admit()) + successor = prepare_authority(store, store.effective_spec(admitted["run_id"])) + broker = DeliveryBroker(store, successor) + broker.prepare() + context = revisions.begin_revision(store, successor) + diagnostic = _revision_diagnostic_for(successor, diagnostic, broker.candidate()) + budget = shared.budget(previous["owner"]["issue_id"]) + assert budget["used"] == 6 + with store._connect() as db: + attempt = dict(db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND role='review'", + (original["run_id"],), + ).fetchone()) + folder = Path(attempt["result_path"]).parent + if early_prelaunch: + assert list(folder.iterdir()) == [] + + def refused(): + with pytest.raises(ValueError, + match="nonprogressing repeated proposal is already rejected"): + revisions.record_proposal(store, successor, context["revision_id"], plan, + diagnostic=diagnostic) + assert shared.budget(previous["owner"]["issue_id"]) == budget + + lease_path = Path(original["state_dir"]) / "attempt-owners" / attempt["job_key"] + assert lease_path.is_file() + lease = DeliverySupervisor._owner_lock(original, attempt["job_key"]) + assert lease is not None + try: + refused() + finally: + os.close(lease) + marker = folder / "launch.json" + marker.write_text("{}") + refused() + marker.unlink() + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET candidate_id=? WHERE job_key=?", + ("forged", attempt["job_key"])) + refused() + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET candidate_id=? WHERE job_key=?", + (attempt["candidate_id"], attempt["job_key"])) + adverse = {**json.loads(attempt["result_json"]), "status": "findings", + "finish_reason": "done"} + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET result_json=? WHERE job_key=?", + (canonical_json(adverse), attempt["job_key"])) + refused() + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET result_json=? WHERE job_key=?", + (attempt["result_json"], attempt["job_key"])) + assert list(folder.iterdir()) == [] + proposal = revisions.record_proposal(store, successor, context["revision_id"], plan, + diagnostic=diagnostic) + assert proposal["unreviewed_predecessors"][0]["revision_id"] == previous["revision_id"] + assert shared.budget(previous["owner"]["issue_id"]) == budget + with store._connect() as db: + assert dict(db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", + (attempt["job_key"],)).fetchone()) == attempt From 49c1375fc70e018e5ade3362fe82d9549e84bac8 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:38:26 +0100 Subject: [PATCH 4/8] fix(recovery): classify early revision prelaunch evidence --- docs/trusted-local-delivery.md | 7 +- .../delivery_feature_revisions.py | 66 +++++++++++-------- .../tests/test_delivery_feature_revisions.py | 10 ++- 3 files changed, 53 insertions(+), 30 deletions(-) diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index 0bf6afc4..348e47c7 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -204,9 +204,10 @@ written, the unreviewed-predecessor check can reconstruct the review identity from the sealed revision, proposal, candidate and owner. It also requires the supervisor's completed prelaunch result, registered workspace, an empty attempt directory and the original unheld owner lease. A missing request alone does not -prove the exception, and a real adverse review remains a rejection. Generic -completed-resource observation still requires its existing request evidence for -this early shape. +prove the exception, and a real adverse review remains a rejection. The strict +reconsideration reader classifies the same evidence as prelaunch, never as an +independent review. Generic completed-resource observation still requires its +existing request evidence for this early shape. An older run stopped by that browser-output registration defect can use one `published_check_prelaunch_retry` with zero additional implementation turns. diff --git a/runtime/src/devflow_temporal/delivery_feature_revisions.py b/runtime/src/devflow_temporal/delivery_feature_revisions.py index c2de5011..f2782db1 100644 --- a/runtime/src/devflow_temporal/delivery_feature_revisions.py +++ b/runtime/src/devflow_temporal/delivery_feature_revisions.py @@ -1652,6 +1652,31 @@ def _private_canonical(path, expected): return hashlib.sha256(encoded).hexdigest() +def _early_revision_prelaunch(store, spec, attempt, expected, *, closed_at=None): + """Observe a sealed revision review that stopped before persisting its envelope.""" + from .delivery_prelaunch_cleanup import completed_prelaunch + from .delivery_resources import RunResources, read_private + + intent = {key: expected[key] for key in ( + "spec", "role", "iteration", "candidate", "resume_session", "revision_context", + "workspace", + )} + manifest = read_private(RunResources(spec).manifest) + with completed_prelaunch(spec, dict(attempt), {Path(spec["state_dir"]): spec}, manifest, + early_revision_intent=intent) as observed: + with store._connect() as db: + current = db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", + (spec["run_id"], attempt["job_key"]), + ).fetchone() + finished = datetime.fromisoformat(attempt["finished_at"]) + if (not current or dict(current) != dict(attempt) or finished.tzinfo is None + or (closed_at is not None and finished > closed_at)): + raise ValueError("early revision attempt changed or did not precede closure") + result = json.loads(attempt["result_json"]) + return result, observed + + def _saved_revision_role(store, spec, context, proposal, candidate, role, *, closed_at=None, shared=None): """Authenticate the exact supervised launcher and actual completed provider.""" @@ -1692,6 +1717,13 @@ def _saved_revision_role(store, spec, context, proposal, candidate, role, *, clo if steering and json.loads(steering[0]): expected["steering"] = json.loads(steering[0]) request_path = folder / "request.json" + if role == "review" and not os.path.lexists(request_path): + result, observed = _early_revision_prelaunch(store, spec, saved, expected, + closed_at=closed_at) + return result, {"run_id": spec["run_id"], "job_key": key, + "intent_sha256": observed["revision_intent_sha256"], + "result_sha256": digest(result), "session_id": None, + "classification": "prelaunch", "prelaunch_evidence": observed} if request_path.resolve(strict=True) != request_path: raise ValueError("historical native request left its exact owned path") request = read_private(request_path) @@ -2269,32 +2301,14 @@ def _unreviewed_rejection(store, spec, context, rejected, values): # attempt under its original owner lease instead. if not attempt: return None - from .delivery_prelaunch_cleanup import completed_prelaunch - from .delivery_resources import RunResources - - intent = {key: expected[key] for key in ( - "spec", "role", "iteration", "candidate", "resume_session", "revision_context", - "workspace", - )} - manifest = read_private(RunResources(historical).manifest) - with completed_prelaunch(historical, dict(attempt), - {Path(historical["state_dir"]): historical}, manifest, - early_revision_intent=intent) as observed: - with store._connect() as db: - current_attempt = db.execute( - "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", - (owner["run_id"], key), - ).fetchone() - if (not current_attempt or dict(current_attempt) != dict(attempt) - or datetime.fromisoformat(attempt["finished_at"]) > closed_at): - return None - result = json.loads(attempt["result_json"]) - proof = { - "revision_id": revision_id, "review_job_key": key, - "review_intent_digest": observed["revision_intent_sha256"], - "review_result_digest": digest(result), "predecessor_closure": closure, - "prelaunch_evidence": observed, - } + result, observed = _early_revision_prelaunch( + store, historical, attempt, expected, closed_at=closed_at) + proof = { + "revision_id": revision_id, "review_job_key": key, + "review_intent_digest": observed["revision_intent_sha256"], + "review_result_digest": digest(result), "predecessor_closure": closure, + "prelaunch_evidence": observed, + } continue if request_path.resolve(strict=True) != request_path: return None diff --git a/runtime/tests/test_delivery_feature_revisions.py b/runtime/tests/test_delivery_feature_revisions.py index 610e5791..1a5fb5af 100644 --- a/runtime/tests/test_delivery_feature_revisions.py +++ b/runtime/tests/test_delivery_feature_revisions.py @@ -615,7 +615,7 @@ def test_public_revision_retains_unreviewed_prelaunch(native_configuration, monk native_configuration[0].raw["execution_mode"] = "trusted-local" fixture = _native_rejected_fixture(native_configuration, monkeypatch, early_prelaunch=early_prelaunch) - store, original, shared, plan, diagnostic, _, previous, _, _, row = fixture + store, original, shared, plan, diagnostic, candidate, previous, prior_proposal, _, row = fixture command = {"command_id": "prelaunch-revision", "expected_revision": row["revision"], "reason": "Retry the correction after the environment refusal."} @@ -649,6 +649,14 @@ async def admit(): ).fetchone()) folder = Path(attempt["result_path"]).parent if early_prelaunch: + from datetime import UTC, datetime + + review_result, review_proof = revisions._saved_revision_role( + store, original, previous, prior_proposal, candidate, "review", + closed_at=datetime(2099, 1, 1, tzinfo=UTC), + ) + assert review_proof["classification"] == "prelaunch" + assert review_result["status"] == "blocked" assert list(folder.iterdir()) == [] def refused(): From 70ce906148a1faaf16a7404b3fa0127a9d689373 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:42:43 +0100 Subject: [PATCH 5/8] fix(recovery): lock prelaunch resource evidence --- .../delivery_feature_revisions.py | 29 ++++++++++--------- .../tests/test_delivery_feature_revisions.py | 25 +++++++++++++--- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/runtime/src/devflow_temporal/delivery_feature_revisions.py b/runtime/src/devflow_temporal/delivery_feature_revisions.py index f2782db1..fdcd9cc0 100644 --- a/runtime/src/devflow_temporal/delivery_feature_revisions.py +++ b/runtime/src/devflow_temporal/delivery_feature_revisions.py @@ -1655,25 +1655,26 @@ def _private_canonical(path, expected): def _early_revision_prelaunch(store, spec, attempt, expected, *, closed_at=None): """Observe a sealed revision review that stopped before persisting its envelope.""" from .delivery_prelaunch_cleanup import completed_prelaunch - from .delivery_resources import RunResources, read_private + from .delivery_resources import RunResources intent = {key: expected[key] for key in ( "spec", "role", "iteration", "candidate", "resume_session", "revision_context", "workspace", )} - manifest = read_private(RunResources(spec).manifest) - with completed_prelaunch(spec, dict(attempt), {Path(spec["state_dir"]): spec}, manifest, - early_revision_intent=intent) as observed: - with store._connect() as db: - current = db.execute( - "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", - (spec["run_id"], attempt["job_key"]), - ).fetchone() - finished = datetime.fromisoformat(attempt["finished_at"]) - if (not current or dict(current) != dict(attempt) or finished.tzinfo is None - or (closed_at is not None and finished > closed_at)): - raise ValueError("early revision attempt changed or did not precede closure") - result = json.loads(attempt["result_json"]) + registry = RunResources(spec, read_only=True) + with registry.locked(read_only=True) as manifest: + with completed_prelaunch(spec, dict(attempt), {Path(spec["state_dir"]): spec}, manifest, + early_revision_intent=intent) as observed: + with store._connect() as db: + current = db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", + (spec["run_id"], attempt["job_key"]), + ).fetchone() + finished = datetime.fromisoformat(attempt["finished_at"]) + if (not current or dict(current) != dict(attempt) or finished.tzinfo is None + or (closed_at is not None and finished > closed_at)): + raise ValueError("early revision attempt changed or did not precede closure") + result = json.loads(attempt["result_json"]) return result, observed diff --git a/runtime/tests/test_delivery_feature_revisions.py b/runtime/tests/test_delivery_feature_revisions.py index 1a5fb5af..417f52b0 100644 --- a/runtime/tests/test_delivery_feature_revisions.py +++ b/runtime/tests/test_delivery_feature_revisions.py @@ -649,12 +649,29 @@ async def admit(): ).fetchone()) folder = Path(attempt["result_path"]).parent if early_prelaunch: + from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime + from threading import Event - review_result, review_proof = revisions._saved_revision_role( - store, original, previous, prior_proposal, candidate, "review", - closed_at=datetime(2099, 1, 1, tzinfo=UTC), - ) + from devflow_temporal.delivery_resources import RunResources + + started = Event() + + def read_review(): + started.set() + return revisions._saved_revision_role( + store, original, previous, prior_proposal, candidate, "review", + closed_at=datetime(2099, 1, 1, tzinfo=UTC), + ) + + with ThreadPoolExecutor(max_workers=1) as executor: + registry = RunResources(original, read_only=True) + with registry.locked(read_only=True, exclusive=True): + pending = executor.submit(read_review) + assert started.wait(5) + with pytest.raises(TimeoutError): + pending.result(timeout=0.2) + review_result, review_proof = pending.result(timeout=15) assert review_proof["classification"] == "prelaunch" assert review_result["status"] == "blocked" assert list(folder.iterdir()) == [] From 882d162190725aadc9a1ff17cc955a9c7812626d Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 11:49:55 +0100 Subject: [PATCH 6/8] fix(recovery): retain mixed revision review history --- docs/trusted-local-delivery.md | 6 +- .../delivery_feature_revisions.py | 9 ++- .../tests/test_delivery_feature_revisions.py | 75 +++++++++++++++++++ 3 files changed, 85 insertions(+), 5 deletions(-) diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index 348e47c7..ef1c0e7e 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -206,8 +206,10 @@ supervisor's completed prelaunch result, registered workspace, an empty attempt directory and the original unheld owner lease. A missing request alone does not prove the exception, and a real adverse review remains a rejection. The strict reconsideration reader classifies the same evidence as prelaunch, never as an -independent review. Generic completed-resource observation still requires its -existing request evidence for this early shape. +independent review. When a later genuine review is reconsidered, its history +retains the earlier prelaunch intent digest without inventing a request digest. +Generic completed-resource observation still requires its existing request +evidence for this early shape. An older run stopped by that browser-output registration defect can use one `published_check_prelaunch_retry` with zero additional implementation turns. diff --git a/runtime/src/devflow_temporal/delivery_feature_revisions.py b/runtime/src/devflow_temporal/delivery_feature_revisions.py index fdcd9cc0..d61b54f2 100644 --- a/runtime/src/devflow_temporal/delivery_feature_revisions.py +++ b/runtime/src/devflow_temporal/delivery_feature_revisions.py @@ -2044,10 +2044,13 @@ def _prove_reconsideration(store, spec, selectors, record, closure, values, *, p _text(text, "unresolved prior finding", maximum=16 * 1024) if text not in findings: findings.append(text) + review = proof["review"] + review_identity = ({"review_request_sha256": review["request_sha256"]} + if "request_sha256" in review else + {"review_intent_sha256": review["intent_sha256"]}) predecessors.append({"revision_id": proof["revision_id"], - "classification": proof["classification"], - "review_request_sha256": proof["review"]["request_sha256"], - "review_result_sha256": proof["review"]["result_sha256"]}) + "classification": proof["classification"], **review_identity, + "review_result_sha256": review["result_sha256"]}) product_findings = [text for worker in subject["worker_summaries"] for text in (worker.get("current_review_subject") or {}).get( "unresolved_product_findings", [])] diff --git a/runtime/tests/test_delivery_feature_revisions.py b/runtime/tests/test_delivery_feature_revisions.py index 417f52b0..d9aad92d 100644 --- a/runtime/tests/test_delivery_feature_revisions.py +++ b/runtime/tests/test_delivery_feature_revisions.py @@ -719,3 +719,78 @@ def refused(): with store._connect() as db: assert dict(db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", (attempt["job_key"],)).fetchone()) == attempt + +def test_public_reconsideration_preserves_early_and_genuine_reviews( + native_configuration, monkeypatch, tmp_path, +): + """An unlaunched review stays unreviewed beside a later genuine rejection.""" + import httpx + + from devflow_temporal.delivery_api import create_app + + native_configuration[0].raw['execution_mode'] = 'trusted-local' + fixture = _native_rejected_fixture(native_configuration, monkeypatch, + retain_worker='build', early_prelaunch=True) + store, original, shared, plan, diagnostic, _, prior, _, rejection, row = fixture + _, _, historical, current = context_repository(tmp_path, monkeypatch) + _complete_native_role_fixture(store, original, prior, plan, diagnostic, 'intake', historical) + command = {'command_id': 'mixed-ordinary-revision', 'expected_revision': row['revision'], + 'reason': 'Retry an unreviewed plan correction.'} + + async def post(run_id, command): + app = create_app(store.config.path) + app.state.delivery.store = store + origin = store.config.dashboard_url.rstrip('/') + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=origin) as c: + csrf = (await c.get('/api/session')).json()['csrf_token'] + return await c.post(f'/api/runs/{run_id}/revise-feature-plan', json=command, + headers={'Origin': origin, 'X-Devflow-CSRF': csrf}) + + early_selectors = {'version': 1, 'kind': 'derivation_context_change', + 'rejected_revision_id': prior['revision_id'], + 'proposal_sha256': digest(plan), + 'expected_runtime_revision': current['revision'], + 'expected_runtime_payload_sha256': current['runtime_payload_sha256']} + early_command = {**command, 'command_id': 'reject-unreviewed-reconsideration', + 'reconsideration': early_selectors} + denied = asyncio.run(post(original['run_id'], early_command)) + assert denied.status_code == 409, denied.text + assert 'genuine completed independent review' in denied.text + assert shared.budget(prior['owner']['issue_id'])['used'] == 5 + response = asyncio.run(post(original['run_id'], command)) + assert response.status_code == 200, response.text + successor = prepare_authority(store, store.effective_spec(response.json()['run_id'])) + broker = DeliveryBroker(store, successor) + broker.prepare() + with store._connect() as db: + db.execute('UPDATE delivery_runs SET candidate_json=? WHERE run_id=?', + (canonical_json(broker.candidate()), successor['run_id'])) + context = revisions.begin_revision(store, successor) + diagnostic = _revision_diagnostic_for(successor, diagnostic, broker.candidate()) + _complete_native_role_fixture(store, successor, context, plan, diagnostic, 'intake', historical) + proposal = revisions.record_proposal(store, successor, context['revision_id'], plan, + diagnostic=diagnostic) + assert proposal['unreviewed_predecessors'][0]['revision_id'] == prior['revision_id'] + _complete_native_role_fixture(store, successor, context, plan, diagnostic, 'review', historical) + rejected = revisions.reject_revision(store, successor, context['revision_id'], + 'A genuine independent review found a defect.') + stopped = _stop_revision_fixture(store, successor, shared) + selectors = {'version': 1, 'kind': 'derivation_context_change', + 'rejected_revision_id': context['revision_id'], 'proposal_sha256': digest(plan), + 'expected_runtime_revision': current['revision'], + 'expected_runtime_payload_sha256': current['runtime_payload_sha256']} + command = {'command_id':'mixed-reconsideration','expected_revision':stopped['revision'], + 'reason':'Reconsider after the verified derivation change.', + 'reconsideration':selectors} + response = asyncio.run(post(successor['run_id'], command)) + assert response.status_code == 200, response.text + proofs = response.json()['reconsideration']['prior_rejections'] + earlier = next(x for x in proofs if x['revision_id'] == prior['revision_id']) + later = next(x for x in proofs if x['revision_id'] == context['revision_id']) + assert earlier['classification'] == 'prelaunch' and len(earlier['review_intent_sha256']) == 64 + assert 'review_request_sha256' not in earlier + assert later['classification'] == 'genuine' and len(later['review_request_sha256']) == 64 + assert response.json()['repair_budget']['used'] == 6 + checkpoints = shared.checkpoints(prior['owner']['issue_id']) + assert checkpoints[revisions.PREFIX + 'rejected:' + prior['revision_id']] == rejection + assert checkpoints[revisions.PREFIX + 'rejected:' + context['revision_id']] == rejected From adb8743a73419b75646fffecae74e7997a61cb1c Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 29 Oct 2026 20:49:40 +0100 Subject: [PATCH 7/8] fix(recovery): back off retained readiness retries --- docs/trusted-local-delivery.md | 6 ++ .../devflow_temporal/delivery_gate_retry.py | 23 +++++- .../delivery_readiness_gate_retry.py | 25 +++++++ ...test_delivery_feature_readiness_handoff.py | 1 + .../test_delivery_readiness_gate_retry.py | 73 ++++++++++++++++++- 5 files changed, 125 insertions(+), 3 deletions(-) diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index ef1c0e7e..7ae0df6f 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -188,6 +188,12 @@ for the unchanged candidate, preserving the old receipt, attempt and effect ledger. Every retry probes readiness again and runs the original frozen gate; its readiness receipt is never a gate pass. Other failed gates, pending product findings, uncertain cleanup, changed source or unsettled effects refuse admission. +The first readiness retry waits 30 seconds after the authenticated predecessor +closes; the second waits 60 seconds after its predecessor closes. A missing or +ambiguous close time is reported as `readiness_retry_clock_unknown`, and an +unexpired delay as `readiness_retry_backoff`. Waiting allocates no new attempt, +admission, effect or product-repair grant. The delay is derived from the closed +workflow, so restarting the control service does not restart or bypass it. The initial route is qualified only for the fake fixture provider: native runs refuse with `native_descendant_settlement_unknown` until a complete descendant settlement contract can be authenticated. An exhausted retry allowance reports diff --git a/runtime/src/devflow_temporal/delivery_gate_retry.py b/runtime/src/devflow_temporal/delivery_gate_retry.py index 21d948f5..4df76f9c 100644 --- a/runtime/src/devflow_temporal/delivery_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_gate_retry.py @@ -897,9 +897,17 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= else historical_teardown or prelaunch_observation or ( _stopped_cleanup(spec, read_only=True) if _observation else _stopped_cleanup(spec))) + readiness_backoff = None if readiness_retry: from .delivery_lifecycle import require_transition - from .delivery_readiness_gate_retry import RETRY_LIMIT + from .delivery_readiness_gate_retry import RETRY_LIMIT, retry_backoff + + remaining = readiness_generations < RETRY_LIMIT + backoff_elapsed = None + if remaining: + readiness_backoff, backoff_elapsed = retry_backoff( + closed.get('closed_at'), readiness_generations + 1) + clock_known = not remaining or backoff_elapsed is not None # An observed native monitor does not prove its complete descendant set. # The fake fixture has no native actor; real execution needs a separate @@ -909,6 +917,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= 'normalized_lifecycle': 'stopped', 'proof_digest': digest({'row': _sealed_row(row), 'cleanup': cleanup, 'readiness': readiness_observation, + 'readiness_backoff': readiness_backoff, 'prior_readiness_retries': readiness_generations}), 'comparisons': [], 'predicates': [ {'code': 'native_descendant_settlement_unknown', @@ -916,9 +925,18 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= 'missing_or_unknown_fact': 'complete native descendant settlement', 'next_action': 'obtain authenticated native descendant settlement'}, {'code': 'readiness_retry_limit_exhausted', - 'satisfied': readiness_generations < RETRY_LIMIT, + 'satisfied': remaining, 'missing_or_unknown_fact': 'remaining infrastructure retry allowance', 'next_action': 'inspect retained failures and reconcile the environment'}, + {'code': 'readiness_retry_clock_unknown', + 'satisfied': True if clock_known else None, + 'missing_or_unknown_fact': 'authenticated predecessor close time', + 'next_action': 'inspect the retained workflow closure'}, + {'code': 'readiness_retry_backoff', + 'satisfied': backoff_elapsed if remaining and clock_known else True, + 'missing_or_unknown_fact': 'elapsed infrastructure retry backoff', + 'next_action': ('retry after ' + readiness_backoff['retry_not_before'] + if remaining and clock_known else 'inspect the retry allowance')}, ], }, {}) result = {'row': _sealed_row(row), 'closed': closed, 'original_spec': spec, @@ -932,6 +950,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= **({'ordinary_prerequisite_observation': prerequisite_observation} if prerequisite_observation else {}), **({'readiness_observation': readiness_observation} if readiness_observation else {}), + **({'readiness_backoff': readiness_backoff} if readiness_backoff else {}), **({'report_observation': report_observation} if report_observation else {}), **({'verification_observation': verification_observation} if verification_observation else {}), diff --git a/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py b/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py index 47cd5ca0..886a7197 100644 --- a/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py @@ -8,15 +8,40 @@ import hashlib import json import re +from datetime import UTC, datetime, timedelta from pathlib import Path from .contracts import digest from .delivery_runtime_location import private_bytes RETRY_LIMIT = 2 +BACKOFF_SECONDS = 30 SLOTS = {"prepublish": 100, "local": 101, "browser_qa": 102} +def _utc_now(): + return datetime.now(UTC) + + +def retry_backoff(closed_at, generation): + """Bind the next retry's delay to the authentic predecessor closure.""" + if type(generation) is not int or not 1 <= generation <= RETRY_LIMIT: + raise ValueError("readiness retry generation exceeds its finite bound") + delay = BACKOFF_SECONDS * 2 ** (generation - 1) + proof = {"closed_at": closed_at, "delay_seconds": delay, "retry_not_before": None} + if not isinstance(closed_at, str): + return proof, None + try: + stopped = datetime.fromisoformat(closed_at) + if stopped.tzinfo is None or stopped.utcoffset() is None: + return proof, None + retry_at = stopped.astimezone(UTC) + timedelta(seconds=delay) + except (ValueError, OverflowError): + return proof, None + proof["retry_not_before"] = retry_at.isoformat() + return proof, _utc_now() >= retry_at + + def probe_attempt(spec, candidate, slot): """Reserve disjoint readiness evidence names for the two retry generations.""" if slot not in SLOTS.values(): diff --git a/runtime/tests/test_delivery_feature_readiness_handoff.py b/runtime/tests/test_delivery_feature_readiness_handoff.py index 37a300aa..ef8420a6 100644 --- a/runtime/tests/test_delivery_feature_readiness_handoff.py +++ b/runtime/tests/test_delivery_feature_readiness_handoff.py @@ -161,6 +161,7 @@ def admit(config, *args, **kwargs): closed = { "workflow_id": row["workflow_id"], "execution_run_id": "closed-worker", + "closed_at": "2020-01-01T00:00:00+00:00", "request_digest": child["request_digest"], "recovery_digest": None, "result": state, diff --git a/runtime/tests/test_delivery_readiness_gate_retry.py b/runtime/tests/test_delivery_readiness_gate_retry.py index 83c12b40..9ee46d01 100644 --- a/runtime/tests/test_delivery_readiness_gate_retry.py +++ b/runtime/tests/test_delivery_readiness_gate_retry.py @@ -10,6 +10,7 @@ import json from contextlib import asynccontextmanager from copy import deepcopy +from datetime import UTC, datetime, timedelta from pathlib import Path import httpx @@ -169,6 +170,7 @@ def failure(execution, current): closed = { "workflow_id": "delivery-run-1", "execution_run_id": "closed-original", + "closed_at": "2020-01-01T00:00:00+00:00", "request_digest": spec["request_digest"], "recovery_digest": None, "result": state, @@ -194,8 +196,15 @@ async def client_for(store): @pytest.mark.asyncio -async def test_public_retry_preserves_receipts_and_bounds_repeated_failures(stopped_readiness): +async def test_public_retry_preserves_receipts_and_bounds_repeated_failures( + stopped_readiness, monkeypatch +): store, broker, state, command, original, closed, failure = stopped_readiness + from devflow_temporal import delivery_readiness_gate_retry + + clock = [datetime(2020, 1, 1, tzinfo=UTC)] + monkeypatch.setattr(delivery_readiness_gate_retry, "_utc_now", lambda: clock[0], raising=False) + closed["closed_at"] = clock[0].isoformat() candidate = deepcopy(state["candidate"]) retained = Path(original["readiness"]["receipt"]).read_bytes() with store._connect() as db: @@ -203,7 +212,64 @@ async def test_public_retry_preserves_receipts_and_bounds_repeated_failures(stop initial_effects = [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] attempts = [original["readiness"]["attempt"]] async with client_for(store) as (client, headers): + # A missing close time cannot grant a retry or silently disable backoff. + for invalid_close in (None, "2020-01-01T00:00:00"): + closed["closed_at"] = invalid_close + assessment = await client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert assessment.status_code == 200 + assert any(item["code"] == "readiness_retry_clock_unknown" + and item["satisfied"] is None + for item in assessment.json()["blockers"]) + closed["closed_at"] = clock[0].isoformat() for generation in (1, 2): + # Infrastructure retries wait 30 seconds and then 60 seconds; + # no product allowance or attempt is allocated during the wait. + waiting = await client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert waiting.status_code == 200 + assert any(item["code"] == "readiness_retry_backoff" + for item in waiting.json()["blockers"]) + if generation == 2: + # Restarting the control service must not reset the delay. + restarted_store = DeliveryStore(DeliveryConfig.load(store.config.path)) + async with client_for(restarted_store) as (restarted_client, _): + after_restart = await restarted_client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert after_restart.status_code == 200 + assert any(item["code"] == "readiness_retry_backoff" + for item in after_restart.json()["blockers"]) + with store._connect() as db: + admissions = [dict(x) for x in db.execute("SELECT * FROM delivery_gate_admissions")] + denied = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert denied.status_code == 409 + with store._connect() as db: + assert [dict(x) for x in db.execute("SELECT * FROM delivery_attempts")] == ( + initial_attempts + ) + assert [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] == ( + initial_effects + ) + assert [dict(x) for x in db.execute("SELECT * FROM delivery_gate_admissions")] == ( + admissions + ) + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + delay = 30 * 2 ** (generation - 1) + clock[0] += timedelta(seconds=delay - 1) + before_deadline = await client.post( + "/api/runs/run-1/repair-admission-preflight", json=command + ) + assert before_deadline.status_code == 409 + assert "readiness_retry_backoff" in before_deadline.text + clock[0] += timedelta(seconds=1) preflight = await client.post( "/api/runs/run-1/repair-admission-preflight", json=command ) @@ -237,6 +303,10 @@ async def test_public_retry_preserves_receipts_and_bounds_repeated_failures(stop assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() assert execution["gate_retry_generation"] == generation assert execution["gate_retry_stage"] == "readiness" + assert recovery["seal"]["readiness_backoff"]["delay_seconds"] == delay + assert recovery["seal"]["readiness_backoff"]["retry_not_before"] == ( + clock[0].isoformat() + ) assert recovery["state"]["checks"]["prepublish"]["readiness"]["state"] == "failed" assert readback(store, execution, recovery) is None if generation == 1: @@ -276,6 +346,7 @@ async def test_public_retry_preserves_receipts_and_bounds_repeated_failures(stop closed.update( workflow_id=admitted["workflow_id"], execution_run_id=f"closed-{generation}", + closed_at=clock[0].isoformat(), recovery_digest=digest(recovery), result=current, ) From 200a4d80dd3614dab27c00a63655ab5c7d41c900 Mon Sep 17 00:00:00 2001 From: Eloi Date: Sun, 1 Nov 2026 19:46:30 +0100 Subject: [PATCH 8/8] ci: validate retry on its final stack base