diff --git a/docs/trusted-local-delivery.md b/docs/trusted-local-delivery.md index ebee2114..7ae0df6f 100644 --- a/docs/trusted-local-delivery.md +++ b/docs/trusted-local-delivery.md @@ -181,6 +181,42 @@ create its reports before the dedicated browser gate. Nested output parents are created only inside an authenticated checkout. Existing unregistered outputs, tracked contents, protected paths and symlinked parents remain rejected. +A check stopped before launch by execution readiness records a versioned failure +class with its stage, candidate and immutable readiness-receipt digest. The +existing zero-repair gate control can admit at most two fresh readiness retries +for the unchanged candidate, preserving the old receipt, attempt and effect +ledger. Every retry probes readiness again and runs the original frozen gate; +its readiness receipt is never a gate pass. Other failed gates, pending product +findings, uncertain cleanup, changed source or unsettled effects refuse admission. +The first readiness retry waits 30 seconds after the authenticated predecessor +closes; the second waits 60 seconds after its predecessor closes. A missing or +ambiguous close time is reported as `readiness_retry_clock_unknown`, and an +unexpired delay as `readiness_retry_backoff`. Waiting allocates no new attempt, +admission, effect or product-repair grant. The delay is derived from the closed +workflow, so restarting the control service does not restart or bypass it. +The initial route is qualified only for the fake fixture provider: native runs +refuse with `native_descendant_settlement_unknown` until a complete descendant +settlement contract can be authenticated. An exhausted retry allowance reports +`readiness_retry_limit_exhausted`. Neither blocker is a launch grant. A transient +unpublished stop with a recorded passing implementation is retained for an +authenticated same-candidate continuation; the automatic fresh-run path refuses +to replace it with another implementation. For a feature worker, the new +readiness path also requires an already authenticated successor assignment. The +current coordinator examines the stopped worker before reserving that assignment, +so an unreserved continuation remains blocked rather than bypassing ownership. + +For a feature-plan correction stopped before its review launch envelope is +written, the unreviewed-predecessor check can reconstruct the review identity +from the sealed revision, proposal, candidate and owner. It also requires the +supervisor's completed prelaunch result, registered workspace, an empty attempt +directory and the original unheld owner lease. A missing request alone does not +prove the exception, and a real adverse review remains a rejection. The strict +reconsideration reader classifies the same evidence as prelaunch, never as an +independent review. When a later genuine review is reconsidered, its history +retains the earlier prelaunch intent digest without inventing a request digest. +Generic completed-resource observation still requires its existing request +evidence for this early shape. + An older run stopped by that browser-output registration defect can use one `published_check_prelaunch_retry` with zero additional implementation turns. Admission requires its exact closed UNKNOWN checkpoint, passed local checks and diff --git a/runtime/src/devflow_temporal/delivery_activities.py b/runtime/src/devflow_temporal/delivery_activities.py index bca456e6..2f37ec2a 100644 --- a/runtime/src/devflow_temporal/delivery_activities.py +++ b/runtime/src/devflow_temporal/delivery_activities.py @@ -1221,12 +1221,13 @@ def _check_readiness(broker, request, attempt): workspace = broker.gate_checkout("verify", request["iteration"], request["candidate"]) requirements = describe_requirements( request["spec"], workspace, request["spec"].get("accepted_plan", "")) - generation = int(digest({"spec": request["spec"], "candidate": request["candidate"]})[:5], - 16) % 300000 - probe_attempt = 10 + generation * 3 + {100: 0, 101: 1, 102: 2}[attempt] + from .delivery_readiness_gate_retry import SLOTS, failure, probe_attempt + + stage = next(name for name, slot in SLOTS.items() if slot == attempt) + selected_attempt = probe_attempt(request["spec"], request["candidate"], attempt) readiness = prepare_and_probe(broker.store, request["spec"], request["candidate"], requirements, "plan", iteration=request["iteration"], - attempt=probe_attempt, workspace=workspace) + attempt=selected_attempt, workspace=workspace) if readiness.get("state") == "passed": verify_receipt(broker.store, request["spec"], request["candidate"], readiness, requirements) return readiness @@ -1237,7 +1238,8 @@ def _check_readiness(broker, request, attempt): "failure_kind": "readiness_preparation", "launched": False, "cleanup": cleanup, "candidate_id": request["candidate"]["id"], "source_unchanged": broker.candidate() == request["candidate"], - "reason": "execution readiness prevented the check", "readiness": readiness} + "reason": "execution readiness prevented the check", "readiness": readiness, + "failure_class": failure(readiness, stage, request["candidate"]["id"], cleanup)} @activity.defn(name="delivery_checks") diff --git a/runtime/src/devflow_temporal/delivery_automatic_retry.py b/runtime/src/devflow_temporal/delivery_automatic_retry.py index 00295fc5..bf5e01ee 100644 --- a/runtime/src/devflow_temporal/delivery_automatic_retry.py +++ b/runtime/src/devflow_temporal/delivery_automatic_retry.py @@ -554,6 +554,19 @@ def retry_once(store, *, stopped=lambda: False): 'SELECT issue_url FROM delivery_runs')) if count >= spec['policy']['max_attempts'] or unavailable(store, db, spec): continue + roles = result.get('roles') + if not isinstance(roles, list) or any(not isinstance(role, dict) for role in roles): + _record_skip(store, row, ValueError( + 'closed run lacks its complete implementation history')) + continue + if any(role.get('role') == 'implement' and role.get('status') == 'pass' + for role in roles): + # A fresh run would start implementation again and abandon the + # stopped candidate. It is not a same-candidate retry grant. + _record_skip(store, row, ValueError( + 'retained implementation requires an authenticated same-candidate ' + 'continuation; inspect the stopped recovery options')) + continue base = fresh_unpublished_base(store, spec) identity = digest({'run_id': spec['run_id'], 'request_digest': row['request_digest']})[:24] diff --git a/runtime/src/devflow_temporal/delivery_feature_revisions.py b/runtime/src/devflow_temporal/delivery_feature_revisions.py index ebbc87e6..d61b54f2 100644 --- a/runtime/src/devflow_temporal/delivery_feature_revisions.py +++ b/runtime/src/devflow_temporal/delivery_feature_revisions.py @@ -1652,6 +1652,32 @@ def _private_canonical(path, expected): return hashlib.sha256(encoded).hexdigest() +def _early_revision_prelaunch(store, spec, attempt, expected, *, closed_at=None): + """Observe a sealed revision review that stopped before persisting its envelope.""" + from .delivery_prelaunch_cleanup import completed_prelaunch + from .delivery_resources import RunResources + + intent = {key: expected[key] for key in ( + "spec", "role", "iteration", "candidate", "resume_session", "revision_context", + "workspace", + )} + registry = RunResources(spec, read_only=True) + with registry.locked(read_only=True) as manifest: + with completed_prelaunch(spec, dict(attempt), {Path(spec["state_dir"]): spec}, manifest, + early_revision_intent=intent) as observed: + with store._connect() as db: + current = db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND job_key=?", + (spec["run_id"], attempt["job_key"]), + ).fetchone() + finished = datetime.fromisoformat(attempt["finished_at"]) + if (not current or dict(current) != dict(attempt) or finished.tzinfo is None + or (closed_at is not None and finished > closed_at)): + raise ValueError("early revision attempt changed or did not precede closure") + result = json.loads(attempt["result_json"]) + return result, observed + + def _saved_revision_role(store, spec, context, proposal, candidate, role, *, closed_at=None, shared=None): """Authenticate the exact supervised launcher and actual completed provider.""" @@ -1692,6 +1718,13 @@ def _saved_revision_role(store, spec, context, proposal, candidate, role, *, clo if steering and json.loads(steering[0]): expected["steering"] = json.loads(steering[0]) request_path = folder / "request.json" + if role == "review" and not os.path.lexists(request_path): + result, observed = _early_revision_prelaunch(store, spec, saved, expected, + closed_at=closed_at) + return result, {"run_id": spec["run_id"], "job_key": key, + "intent_sha256": observed["revision_intent_sha256"], + "result_sha256": digest(result), "session_id": None, + "classification": "prelaunch", "prelaunch_evidence": observed} if request_path.resolve(strict=True) != request_path: raise ValueError("historical native request left its exact owned path") request = read_private(request_path) @@ -2011,10 +2044,13 @@ def _prove_reconsideration(store, spec, selectors, record, closure, values, *, p _text(text, "unresolved prior finding", maximum=16 * 1024) if text not in findings: findings.append(text) + review = proof["review"] + review_identity = ({"review_request_sha256": review["request_sha256"]} + if "request_sha256" in review else + {"review_intent_sha256": review["intent_sha256"]}) predecessors.append({"revision_id": proof["revision_id"], - "classification": proof["classification"], - "review_request_sha256": proof["review"]["request_sha256"], - "review_result_sha256": proof["review"]["result_sha256"]}) + "classification": proof["classification"], **review_identity, + "review_result_sha256": review["result_sha256"]}) product_findings = [text for worker in subject["worker_summaries"] for text in (worker.get("current_review_subject") or {}).get( "unresolved_product_findings", [])] @@ -2261,6 +2297,23 @@ def _unreviewed_rejection(store, spec, context, rejected, values): str(Path(historical["state_dir"]) / "gates" / "0" / "review")), ) request_path = folder / "request.json" + if role == "review" and not os.path.lexists(request_path): + # The revision and proposal identity were reconstructed from + # immutable coordinator checkpoints above, not from a caller. + # A supervisor may refuse preparation before it persists the + # later native launch envelope, so prove the typed terminal + # attempt under its original owner lease instead. + if not attempt: + return None + result, observed = _early_revision_prelaunch( + store, historical, attempt, expected, closed_at=closed_at) + proof = { + "revision_id": revision_id, "review_job_key": key, + "review_intent_digest": observed["revision_intent_sha256"], + "review_result_digest": digest(result), "predecessor_closure": closure, + "prelaunch_evidence": observed, + } + continue if request_path.resolve(strict=True) != request_path: return None request = read_private(request_path) diff --git a/runtime/src/devflow_temporal/delivery_gate_retry.py b/runtime/src/devflow_temporal/delivery_gate_retry.py index 10d2260c..02a3625e 100644 --- a/runtime/src/devflow_temporal/delivery_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_gate_retry.py @@ -74,8 +74,11 @@ def transient_stop(state): def prepublication_preparation_failed(state): """Recognize the controller's exact preparation stop, never a product finding.""" from .delivery_baseline_contract import preparation_failure + from .delivery_readiness_gate_retry import possible as readiness_possible precheck = state.get('checks', {}).get('prepublish', {}) + if readiness_possible(state) == 'prepublish' and precheck.get('state') == 'failed': + return True # Selection hint only; snapshot authenticates the receipt. prerequisite = preparation_failure(precheck) return (precheck.get('state') == 'failed' and prerequisite is not None and state.get('error') == f'environment preparation failed: {prerequisite}; ' @@ -547,6 +550,11 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= from .delivery_prerequisite_gate_recovery import possible as prerequisite_possible prerequisite_retry = kind == KIND and prerequisite_possible(spec, state) + from .delivery_readiness_gate_retry import possible as readiness_possible + + readiness_stage = readiness_possible(state) + readiness_retry = bool(readiness_stage and ( + (unpublished and readiness_stage == 'prepublish') or kind == KIND)) # A recorded gate verdict stands: a transport or controller stop after it is # never a fresh assessment of the same head. verdict = gate_failed(state.get('checks') or {}) @@ -587,7 +595,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= and previous.get('kind') == PREPUBLICATION_KIND and spec.get('gate_retry_generation') == 1) history = previous - ci_observations = transient_generations = 0 + ci_observations = transient_generations = readiness_generations = 0 published_after_recovery = bool(not unpublished and previous and previous.get('kind') in { PREPUBLICATION_KIND, 'pending_publication_retry', 'repair_continuation', 'published_head_validation'}) @@ -595,6 +603,8 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= 'finalized_checkpoint'): published_after_recovery = False while history: + if history.get('seal', {}).get('readiness_observation'): + readiness_generations += 1 if prerequisite_retry and history.get('seal', {}).get('ordinary_prerequisite_observation'): raise ValueError('ordinary prerequisite already received its one gate recovery') if browser_prelaunch and history.get('kind') == PRELAUNCH_KIND: @@ -613,7 +623,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= if transient_retry and transient_generations >= TRANSIENT_GATE_RETRIES: raise ValueError('this run already received its bounded transient gate retries') if ((renewed or published_after_recovery or prelaunch or report_retry or controller_only) - and spec['provider'] != 'fake'): + and not readiness_retry and spec['provider'] != 'fake'): from .delivery_native_preparation import native_identity if native_identity(spec)['runtime_payload_sha256'] in _consumed_payloads( spec, state, previous, include_implementation=controller_only): @@ -672,6 +682,31 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= browser_artifact_observation = None verification_observation = None prerequisite_observation = None + readiness_observation = None + if readiness_retry: + from .delivery_readiness_gate_retry import observe + + if state_checks != row_checks: + raise ValueError('readiness retry lost its exact original checks') + if spec.get('feature_worker'): + from .delivery_feature_execution import require_worker_handoff + + require_worker_handoff(store, spec) + readiness_observation = observe(spec, state) + latest = next((r for r in reversed(roles) if r.get('role') == 'implement'), None) + matches = [a for a in attempts if latest and a['role'] == 'implement' + and a['iteration'] == row['iteration'] + and a['session_id'] == latest.get('session_id')] + from .delivery_stopped_resume import _implementation_receipt + + if (not latest or latest.get('status') != 'pass' or latest.get('findings') + or latest.get('cleanup') != 'confirmed' or not latest.get('session_id') + or len(matches) != 1 or not _implementation_receipt(spec, latest, matches[0]) + or any((latest.get('candidate') or {}).get(key) != candidate.get(key) + for key in ('content_sha256', 'base_sha', 'environment_digest', + 'policy_digest'))): + raise ValueError('readiness retry lacks its exact passing implementation receipt') + failed_gate = True if prerequisite_retry: from .delivery_prerequisite_gate_recovery import observe @@ -776,7 +811,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= or (previous is not None and not ( renewed or published_after_recovery or prelaunch or report_retry or ci_only or transient_retry or attribution_retry or verification_retry - or browser_artifact_retry or prerequisite_retry)) + or browser_artifact_retry or prerequisite_retry or readiness_retry)) or row['phase'] != 'blocked' or row['outcome'] != 'blocked' or row['execution_state'] != 'blocked' or state.get('phase') != 'blocked' or state.get('outcome') != 'blocked' @@ -789,7 +824,7 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= and ((state.get('cleanup') == 'unknown') if prelaunch else (state.get('cleanup') == 'confirmed' or reconciled is not None))) verdict_matches = ( - (transient_retry or transient_ci or masked_exhaustion + (transient_retry or transient_ci or masked_exhaustion or readiness_retry or state.get('error') in ({'repair limit exhausted', 'required CI did not confirm this PR head'} if report_retry else {'required CI did not confirm this PR head' if ci_only @@ -899,7 +934,8 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= ancestor = ancestor.get('original_recovery') if admitted and (not (renewed or published_after_recovery or prelaunch or report_retry or ci_only or transient_retry or attribution_retry - or verification_retry or browser_artifact_retry or prerequisite_retry) + or verification_retry or browser_artifact_retry or prerequisite_retry + or readiness_retry) or (digest(prior_gate) != attribution_observation['prior_gate_digest'] if attribution_retry else prior_gate != ancestor)): raise ValueError('this run already received its bounded gate assessment retry') @@ -911,21 +947,66 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= store, spec, previous, candidate, publication, _inventory_profile) elif _inventory_profile is not None: raise ValueError('inventory physical custody belongs only to zero-model CI') + cleanup = (prerequisite_observation['resources'] if prerequisite_retry + else verification_observation['resources'] if verification_retry + else report_observation['resources'] if layout_report + else historical_teardown or prelaunch_observation or ( + _stopped_cleanup(spec, read_only=True) if _observation + else _stopped_cleanup(spec))) + readiness_backoff = None + if readiness_retry: + from .delivery_lifecycle import require_transition + from .delivery_readiness_gate_retry import RETRY_LIMIT, retry_backoff + + remaining = readiness_generations < RETRY_LIMIT + backoff_elapsed = None + if remaining: + readiness_backoff, backoff_elapsed = retry_backoff( + closed.get('closed_at'), readiness_generations + 1) + clock_known = not remaining or backoff_elapsed is not None + + # An observed native monitor does not prove its complete descendant set. + # The fake fixture has no native actor; real execution needs a separate + # settlement contract before this newly introduced route can launch work. + require_transition({ + 'operation': kind, 'owner': 'delivery_gate_retry', 'owner_proof': True, + 'normalized_lifecycle': 'stopped', + 'proof_digest': digest({'row': _sealed_row(row), 'cleanup': cleanup, + 'readiness': readiness_observation, + 'readiness_backoff': readiness_backoff, + 'prior_readiness_retries': readiness_generations}), + 'comparisons': [], 'predicates': [ + {'code': 'native_descendant_settlement_unknown', + 'satisfied': True if spec['provider'] == 'fake' else None, + 'missing_or_unknown_fact': 'complete native descendant settlement', + 'next_action': 'obtain authenticated native descendant settlement'}, + {'code': 'readiness_retry_limit_exhausted', + 'satisfied': remaining, + 'missing_or_unknown_fact': 'remaining infrastructure retry allowance', + 'next_action': 'inspect retained failures and reconcile the environment'}, + {'code': 'readiness_retry_clock_unknown', + 'satisfied': True if clock_known else None, + 'missing_or_unknown_fact': 'authenticated predecessor close time', + 'next_action': 'inspect the retained workflow closure'}, + {'code': 'readiness_retry_backoff', + 'satisfied': backoff_elapsed if remaining and clock_known else True, + 'missing_or_unknown_fact': 'elapsed infrastructure retry backoff', + 'next_action': ('retry after ' + readiness_backoff['retry_not_before'] + if remaining and clock_known else 'inspect the retry allowance')}, + ], + }, {}) result = {'row': _sealed_row(row), 'closed': closed, 'original_spec': spec, 'attempts': attempts, 'effects': effects, 'candidate': candidate, 'publication': publication, - 'cleanup': (prerequisite_observation['resources'] if prerequisite_retry - else verification_observation['resources'] if verification_retry - else report_observation['resources'] if layout_report - else historical_teardown or prelaunch_observation or ( - _stopped_cleanup(spec, read_only=True) if _observation - else _stopped_cleanup(spec))), + 'cleanup': cleanup, 'work_binding': binding, 'previous_digest': digest(previous) if previous else None, 'prior_gate_digest': digest(prior_gate) if prior_gate else None, **({'local_download_retry': True} if local_download_retry else {}), **({'ordinary_prerequisite_observation': prerequisite_observation} if prerequisite_observation else {}), + **({'readiness_observation': readiness_observation} if readiness_observation else {}), + **({'readiness_backoff': readiness_backoff} if readiness_backoff else {}), **({'report_observation': report_observation} if report_observation else {}), **({'verification_observation': verification_observation} if verification_observation else {}), @@ -938,13 +1019,14 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= if attribution_retry or layout_report or verification_retry else {}), **({'browser_prelaunch_observation': browser_observation} if browser_observation else {}), - 'stage': 'prerequisite' if prerequisite_retry + 'stage': 'readiness' if readiness_retry else 'prerequisite' if prerequisite_retry else 'transient' if transient_retry or verification_retry else 'report' if report_retry else 'ci' if ci_only else 'published' if published_after_recovery or prelaunch or controller_only or attribution_retry or browser_artifact_retry else None, - 'generation': (transient_generations + 1 if transient_retry + 'generation': (readiness_generations + 1 if readiness_retry + else transient_generations + 1 if transient_retry else ci_observations + 1 if ci_only else (2 if previous else 1) if attribution_retry or browser_artifact_retry @@ -1003,6 +1085,8 @@ def admit(store, run_id, payload, *, preflight=False): return json.loads(prior['response_json']) seal = snapshot(store, run_id, kind, _observation=preflight) previous = _retained_recovery(store, run_id, seal) + if seal.get('readiness_observation') and 'verification_test_paths' in payload: + raise ValueError('readiness retry must retain the original frozen test selectors') if 'verification_test_paths' in payload: from .delivery_plan_checks import planned_checks @@ -1037,7 +1121,7 @@ def admit(store, run_id, payload, *, preflight=False): ordering = (seal.get('browser_artifact_observation') or {}).get( 'cause') == 'browser_artifact_assessment_order' if (preflight_generation or seal.get('preparation_observation') - or layout or verification or ordering) \ + or seal.get('readiness_observation') or layout or verification or ordering) \ and spec.get('feature_worker'): from .delivery_feature_execution import require_worker_handoff @@ -1087,7 +1171,8 @@ def admit(store, run_id, payload, *, preflight=False): execution['implementation_role_home_generation'] = implementation_generation( spec, previous) execution['role_home_generation'] = ( - f'report-retry-{generation}' if seal['stage'] == 'report' + f'readiness-retry-{generation}' if seal['stage'] == 'readiness' + else f'report-retry-{generation}' if seal['stage'] == 'report' else f'transient-gate-retry-{generation}' if seal['stage'] == 'transient' else f'gate-retry-{generation}') execution['gate_retry_generation'] = generation @@ -1136,7 +1221,9 @@ def admit(store, run_id, payload, *, preflight=False): else {**seal['publication'], 'candidate': after})} _immutable(root / 'admission.json', recovery) preserve_resources(root, spec) - workflow_id = (f'delivery-{run_id}-ci-retry-{generation}' if kind == CI_KIND else + workflow_id = (f'delivery-{run_id}-readiness-gates-retry-{generation}' + if seal['stage'] == 'readiness' else + f'delivery-{run_id}-ci-retry-{generation}' if kind == CI_KIND else f'delivery-{run_id}-report-gates-retry-{generation}' if seal['stage'] == 'report' else f'delivery-{run_id}-transient-gates-retry-{generation}' @@ -1158,19 +1245,19 @@ def admit(store, run_id, payload, *, preflight=False): if canonical_json(_sealed_row(row)) != canonical_json(seal['row']): raise ValueError('gate retry admission lost its stopped projection') if (verification or ordering or seal.get('ordinary_prerequisite_observation') - or seal.get('ordinary_ci_inventory')): + or seal.get('ordinary_ci_inventory') or seal.get('readiness_observation')): if store.state.claim_for(db, spec['work_id']) is not None: - raise ValueError('listener verification admission lost its released claim') + raise ValueError('gate retry admission lost its released claim') for table, key, expected in [('delivery_attempts', 'job_key', seal['attempts']), ('delivery_effects', 'effect_key', seal['effects'])]: records = [dict(item) for item in db.execute( f'SELECT * FROM {table} WHERE run_id=? ORDER BY {key}', (run_id,))] if records != expected: - raise ValueError('listener verification admission lost its full ledger') + raise ValueError('gate retry admission lost its full ledger') if work_binding(store, spec, db) != seal['work_binding']: raise ValueError('gate retry admission lost issue ownership') if ((preflight_generation or seal.get('preparation_observation') - or layout or verification or ordering) + or seal.get('readiness_observation') or layout or verification or ordering) and spec.get('feature_worker')): require_worker_handoff(store, spec) if preparation_owner is not None: @@ -1236,6 +1323,8 @@ def gate_namespace(generation, stage=None, *, preflight_generation=0, prepared_h suffix = f'-{preflight_generation}' if preflight_generation > 1 else '' return gate_namespace(generation, stage) + '/preflight-continuation' + suffix if stage is not None: + if stage == 'readiness' and type(generation) is int and generation in (1, 2): + return f'readiness-gates-admission-{generation}' if stage == 'prerequisite' and type(generation) is int and generation == 1: return 'ordinary-prerequisite-gates-admission' if stage == 'preparation' and type(generation) is int and generation in (1, 2): @@ -1291,6 +1380,17 @@ def readback(store, spec, recovery, *, resource_spec=None): if resource_spec is not None else DeliveryBroker(store, spec)) if effective != spec or broker.candidate() != recovery['candidate']: raise ValueError('gate retry source or execution authority changed') + if expected := recovery['seal'].get('readiness_observation'): + from .delivery_readiness_gate_retry import observe + + seal = recovery['seal'] + _, attempts, effects, claim = _rows(store, spec['run_id']) + closed = store._completed_temporal_result( + spec['run_id'], workflow_id=seal['closed']['workflow_id']) + if (closed != seal['closed'] or attempts != seal['attempts'] or effects != seal['effects'] + or claim is None or claim['owner'] != f"external:devflow:{spec['run_id']}" + or observe(seal['original_spec'], recovery['state']) != expected): + raise ValueError('readiness retry original evidence or ledger changed') if recovery['seal'].get('ordinary_ci_inventory'): from .delivery_ci_inventory import validate diff --git a/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py b/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py index 8d92dbe9..08ff900d 100644 --- a/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py +++ b/runtime/src/devflow_temporal/delivery_prelaunch_cleanup.py @@ -24,12 +24,15 @@ def _stamp(info): @contextmanager -def completed_prelaunch(spec, attempt, executions, manifest, *, completed_intakes=()): +def completed_prelaunch(spec, attempt, executions, manifest, *, completed_intakes=(), + early_revision_intent=None): """Authenticate the existing supervisor's typed pre-monitor failure, never settle it. The durable result is server-owned producer evidence. Neither a missing journal nor a null session suffices. Existing owner/lifecycle leases exclude an unfinished launch; this observer never creates a missing proof or lock. + An early revision identity must be reconstructed by the owning caller from + the sealed predecessor and proposal before it is passed here. """ from .supervisor import DeliverySupervisor @@ -103,20 +106,36 @@ def lease(path, parent): else: locks[name] = lease(folder / name, folder_fd) - fd = os.open('request.json', os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, - dir_fd=folder_fd) - descriptors.append(fd) - info = os.fstat(fd) - if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() - or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 - or info.st_size > _LIMIT): - raise ValueError('completed prelaunch request is not bounded private evidence') - pinned.append((folder / 'request.json', fd, _stamp(info))) - with os.fdopen(os.dup(fd), 'rb') as stream: - raw = stream.read(_LIMIT + 1) - if len(raw) != info.st_size: - raise ValueError('completed prelaunch request changed during observation') - request = json.loads(raw) + early = False + try: + fd = os.open('request.json', os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, + dir_fd=folder_fd) + except FileNotFoundError: + # Only the revision owner may supply a reconstructed identity after + # authenticating its sealed predecessor, proposal and owner records. + # The exact typed supervisor result and the held owner lease above + # establish no launch; an absent request is never sufficient. + if early_revision_intent is None or os.listdir(folder_fd): + raise + early = True + request = early_revision_intent + raw = None + if (not isinstance(request, dict) or request.get('role') != 'review' + or not isinstance(request.get('revision_context'), dict)): + raise ValueError('early prelaunch requires a sealed review identity') from None + else: + descriptors.append(fd) + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() + or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 + or info.st_size > _LIMIT): + raise ValueError('completed prelaunch request is not bounded private evidence') + pinned.append((folder / 'request.json', fd, _stamp(info))) + with os.fdopen(os.dup(fd), 'rb') as stream: + raw = stream.read(_LIMIT + 1) + if len(raw) != info.st_size: + raise ValueError('completed prelaunch request changed during observation') + request = json.loads(raw) if (not isinstance(request, dict) or not isinstance(request.get('spec'), dict) or not isinstance(request.get('candidate'), dict) or not isinstance(request['candidate'].get('id'), str) @@ -136,13 +155,13 @@ def lease(path, parent): if (not admitted or execution.get('provider') != 'codex' or execution.get('policy', {}).get('execution_backend') != 'native-macos' or execution.get('policy_digest') != digest(execution['policy']) - or request.get('native_authorized') is not True + or (not early and request.get('native_authorized') is not True) or DeliverySupervisor._job_key(request) != key or request.get('role') != attempt['role'] or request.get('iteration') != attempt['iteration'] or request['candidate']['id'] != attempt['candidate_id'] - or request.get('result_path') != attempt['result_path'] - or request.get('start_path') != str(folder / 'start.json') + or (not early and (request.get('result_path') != attempt['result_path'] + or request.get('start_path') != str(folder / 'start.json'))) or role not in execution['policy']['roles'] or type(iteration) is not int or iteration < 0 or workspace not in expected_workspaces @@ -154,6 +173,8 @@ def lease(path, parent): raise ValueError('completed prelaunch lost its original admitted request') def unchanged(): + if early and os.listdir(folder_fd): + raise ValueError('early prelaunch attempt is no longer empty') for path, descriptor, stamp in pinned: current, named = os.fstat(descriptor), path.lstat() if any(getattr(current, k) != v or getattr(named, k) != v @@ -172,7 +193,8 @@ def unchanged(): unchanged() yield {'kind': 'completed_native_prelaunch', 'version': 1, 'attempt_sha256': digest(attempt), 'result_sha256': digest(result), - 'request_sha256': hashlib.sha256(raw).hexdigest(), 'leases': locks} + **({'revision_intent_sha256': digest(request)} if early else + {'request_sha256': hashlib.sha256(raw).hexdigest()}), 'leases': locks} unchanged() finally: for fd in reversed(descriptors): diff --git a/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py b/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py new file mode 100644 index 00000000..886a7197 --- /dev/null +++ b/runtime/src/devflow_temporal/delivery_readiness_gate_retry.py @@ -0,0 +1,152 @@ +"""Authenticate an unlaunched gate readiness refusal for bounded reassessment. + +The immutable refusal is historical evidence, not a readiness pass or a launch +permission. The admitted continuation must probe again before any required gate. +""" +from __future__ import annotations + +import hashlib +import json +import re +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from .contracts import digest +from .delivery_runtime_location import private_bytes + +RETRY_LIMIT = 2 +BACKOFF_SECONDS = 30 +SLOTS = {"prepublish": 100, "local": 101, "browser_qa": 102} + + +def _utc_now(): + return datetime.now(UTC) + + +def retry_backoff(closed_at, generation): + """Bind the next retry's delay to the authentic predecessor closure.""" + if type(generation) is not int or not 1 <= generation <= RETRY_LIMIT: + raise ValueError("readiness retry generation exceeds its finite bound") + delay = BACKOFF_SECONDS * 2 ** (generation - 1) + proof = {"closed_at": closed_at, "delay_seconds": delay, "retry_not_before": None} + if not isinstance(closed_at, str): + return proof, None + try: + stopped = datetime.fromisoformat(closed_at) + if stopped.tzinfo is None or stopped.utcoffset() is None: + return proof, None + retry_at = stopped.astimezone(UTC) + timedelta(seconds=delay) + except (ValueError, OverflowError): + return proof, None + proof["retry_not_before"] = retry_at.isoformat() + return proof, _utc_now() >= retry_at + + +def probe_attempt(spec, candidate, slot): + """Reserve disjoint readiness evidence names for the two retry generations.""" + if slot not in SLOTS.values(): + raise ValueError("unsupported gate readiness slot") + if spec.get("gate_retry_stage") == "readiness": + generation = spec.get("gate_retry_generation") + if type(generation) is not int or not 1 <= generation <= RETRY_LIMIT: + raise ValueError("readiness retry generation exceeds its finite bound") + # Existing hash-derived attempts are in [10, 900009]. + return 900010 + (generation - 1) * 3 + slot - 100 + generation = int(digest({"spec": spec, "candidate": candidate})[:5], 16) % 300000 + return 10 + generation * 3 + slot - 100 + + +def failure(readiness, stage, candidate_id, cleanup): + state = readiness.get("state") + classification = ("environment_prelaunch" if state == "failed" and cleanup == "confirmed" + else "cancelled" if state == "cancelled" else "unknown") + return {"version": 1, "class": classification, "source": "execution_readiness", + "stage": stage, "candidate_id": candidate_id, + "receipt_sha256": readiness.get("receipt_sha256"), "launched": False} + + +def possible(state): + """Selection hint only. `observe` authenticates the result and its receipt.""" + checks = state.get("checks") or {} + if not isinstance(checks, dict): + return None + found = [stage for stage in SLOTS if isinstance(checks.get(stage), dict) + and checks[stage].get("failure_kind") == "readiness_preparation" + and checks[stage].get("launched") is False] + return found[0] if len(found) == 1 else None + + +def observe(spec, state): + """Bind a completed controller result to its original private failure receipt.""" + stage = possible(state) + checks, candidate = state.get("checks") or {}, state.get("candidate") or {} + roles = state.get("roles") + if (not isinstance(roles, list) or any(not isinstance(role, dict) for role in roles)): + raise ValueError("readiness retry lacks its complete role history") + if (spec.get("readiness_version") != 1 or stage is None + or type(state.get("iteration")) is not int or state["iteration"] < 0 + or not isinstance(candidate, dict) or not candidate.get("id")): + raise ValueError("readiness retry requires an exact unlaunched gate refusal") + expected_error = {"prepublish": "execution readiness prevented prepublication", + "local": "execution readiness prevented local checks", + "browser_qa": "execution readiness prevented browser QA"}[stage] + if state.get("error") != expected_error: + raise ValueError("readiness retry lost its exact controller stop") + result = checks[stage] + receipt = result.get("readiness") + if (not isinstance(receipt, dict) or result.get("state") != "failed" + or result.get("cleanup") != "confirmed" or result.get("source_unchanged") is not True + or result.get("candidate_id") != candidate["id"] + or checks.get("readiness") != receipt or receipt.get("version") != 1 + or receipt.get("stage") != "plan" or receipt.get("state") != "failed" + or receipt.get("iteration") != state["iteration"] + or receipt.get("attempt") != probe_attempt(spec, candidate, SLOTS[stage])): + raise ValueError("readiness retry lacks a confirmed candidate-bound refusal") + provenance = failure(receipt, stage, candidate["id"], "confirmed") + if result.get("failure_class") not in (None, provenance): + raise ValueError("readiness failure provenance changed") + # A failed or unassessed product gate remains authoritative. Never replace + # pending findings with a same-candidate retry. + for name in ("prepublish", "local", "browser_qa", "review", "qa", "ci"): + item = checks.get(name) + if name != stage and isinstance(item, dict) and item.get("state") != "passed": + raise ValueError("readiness retry cannot replace another gate verdict") + if any(role.get("iteration") == state["iteration"] + and role.get("role") in {"review", "verify"} + and (role.get("status") != "pass" or role.get("findings")) + for role in roles): + raise ValueError("readiness retry cannot replace an independent product finding") + pending = state.get("repair_findings") or {} + if not isinstance(pending, dict): + raise ValueError("readiness retry lacks its pending finding history") + if pending.get("iteration") == state["iteration"] and pending.get("findings"): + raise ValueError("readiness retry cannot discard pending product findings") + subject = receipt.get("binding") or {} + observations = receipt.get("observations") + if (not isinstance(subject, dict) or subject.get("version") != 1 + or subject.get("spec_sha256") != digest(spec) or subject.get("candidate") != candidate + or not isinstance(subject.get("requirements_sha256"), str) + or not re.fullmatch(r"[0-9a-f]{64}", subject["requirements_sha256"]) + or type(subject.get("resource_generation")) is not int + or subject["resource_generation"] < 0 or not isinstance(subject.get("resources"), dict) + or not isinstance(observations, dict) or not observations + or not any(isinstance(item, dict) and item.get("state") == "failed" + for item in observations.values()) + or any(not isinstance(item, dict) or not isinstance(item.get("state"), str) + or item.get("state") not in {"passed", "failed"} + or item.get("cleanup") != "confirmed" or item.get("candidate") != candidate + or item.get("requirement_sha256") != subject["requirements_sha256"] + for item in observations.values())): + raise ValueError("readiness retry lacks complete confirmed failure observations") + path = (Path(spec["state_dir"]) / "readiness" / "receipts" / "plan" + / str(state["iteration"]) / f"{receipt['attempt']}.json") + if receipt.get("receipt") != str(path): + raise ValueError("readiness failure receipt left its owned run") + raw = private_bytes(path, maximum=8 * 1024 * 1024) + if (hashlib.sha256(raw).hexdigest() != receipt.get("receipt_sha256") + or json.loads(raw) != {key: value for key, value in receipt.items() + if key not in {"receipt", "receipt_sha256"}}): + raise ValueError("readiness failure receipt changed") + return {"stage": stage, "candidate_id": candidate["id"], "attempt": receipt["attempt"], + "receipt": str(path), "receipt_sha256": receipt["receipt_sha256"], + "requirements_sha256": subject["requirements_sha256"], "failure_class": provenance} diff --git a/runtime/src/devflow_temporal/delivery_workflow.py b/runtime/src/devflow_temporal/delivery_workflow.py index 3ddb186e..80f90ed3 100644 --- a/runtime/src/devflow_temporal/delivery_workflow.py +++ b/runtime/src/devflow_temporal/delivery_workflow.py @@ -2664,6 +2664,16 @@ async def _run_iterations( return await self._stop(spec, 'technical prepublication checks failed: ' + type(exc).__name__, cause=exc) self.state['checks']['prepublish'] = checked + if (spec.get('gate_retry_stage') == 'readiness' + and spec.get('readiness_version') == 1 + and checked.get('readiness')): + self.state['checks']['readiness'] = checked['readiness'] + if (checked.get('failure_kind') == 'readiness_preparation' + and checked.get('launched') is False): + if checked.get('cleanup') == 'unknown': + self.state['cleanup'] = 'unknown' + return await self._stop( + spec, 'execution readiness prevented prepublication') if checked.get('state') != 'passed' or checked.get('cleanup') == 'unknown': return await self._stop(spec, 'technical prepublication checks failed') elif recovery is not None and iteration == start_iteration: diff --git a/runtime/tests/test_delivery_automatic_retry.py b/runtime/tests/test_delivery_automatic_retry.py index 08bf9ff7..b9b67d66 100644 --- a/runtime/tests/test_delivery_automatic_retry.py +++ b/runtime/tests/test_delivery_automatic_retry.py @@ -30,7 +30,7 @@ def stopped(service, monkeypatch): 'closed_at': '2026-10-07T00:00:00+00:00', 'request_digest': spec['request_digest'], 'recovery_digest': None, 'result': {'run_id': request['run_id'], 'outcome': 'blocked', 'phase': 'blocked', 'execution_state': 'blocked', - 'cleanup': 'confirmed', 'checks': checks}} + 'cleanup': 'confirmed', 'checks': checks, 'roles': []}} monkeypatch.setattr(DeliveryStore, '_completed_temporal_result', lambda *_a, **_k: closed) monkeypatch.setattr(retry, 'observe_finalized_resources', lambda *_a, **_k: {'finalization_sha256': 'fixture-finalization'}) @@ -150,3 +150,16 @@ def test_automatic_generations_spend_one_original_total_ceiling(service, monkeyp assert retry.retry_once(store) == [] with store._connect() as db: assert db.execute('SELECT COUNT(*) FROM delivery_runs').fetchone()[0] == 3 + +def test_transient_stop_after_implementation_retains_candidate_instead_of_fresh_run( + service, monkeypatch, +): + store, request, closed = stopped(service, monkeypatch) + closed['result'].update(candidate={'id': 'retained-candidate'}, iteration=0, roles=[{ + 'role': 'implement', 'iteration': 0, 'status': 'pass', 'session_id': 'retained-session'}]) + assert retry.retry_once(store) == [] + with store._connect() as db: + assert db.execute('SELECT COUNT(*) FROM delivery_runs').fetchone()[0] == 1 + events = [json.loads(row[0]) for row in db.execute( + "SELECT payload_json FROM delivery_events WHERE type='automatic_retry_skipped'")] + assert any('same-candidate continuation' in event['reason'] for event in events) diff --git a/runtime/tests/test_delivery_feature_readiness_handoff.py b/runtime/tests/test_delivery_feature_readiness_handoff.py new file mode 100644 index 00000000..ef8420a6 --- /dev/null +++ b/runtime/tests/test_delivery_feature_readiness_handoff.py @@ -0,0 +1,196 @@ +"""Observe the feature-worker readiness handoff boundary without a provider.""" + +import json +from copy import deepcopy + +import pytest +from test_delivery_feature_execution import feature_service +from test_delivery_store import service as service + +from devflow_temporal import delivery_feature_activities as activities +from devflow_temporal.contracts import canonical_json, digest +from devflow_temporal.delivery_activities import _check_readiness +from devflow_temporal.delivery_broker import DeliveryBroker +from devflow_temporal.delivery_config import DeliveryConfig +from devflow_temporal.delivery_execution_registry import OwnershipConflict +from devflow_temporal.delivery_feature_execution import ( + register_worker, + registry, + worker_key, + worker_spec, +) +from devflow_temporal.delivery_github_contract import ordered_chunks +from devflow_temporal.delivery_resources import RunResources + + +def test_feature_readiness_requires_reserved_successor_and_preserves_budget(service, monkeypatch): + store, request, _ = feature_service(service, monkeypatch) + original_admit = DeliveryConfig.admit + + def admit(config, *args, **kwargs): + spec = original_admit(config, *args, **kwargs) + spec["readiness_version"] = 1 + spec["policy"]["readiness"] = { + "version": 1, + "generation": 0, + "browser_engines": [], + "temporal_checks": [], + } + spec["policy_digest"] = digest(spec["policy"]) + return spec + + monkeypatch.setattr(DeliveryConfig, "admit", admit) + store.submit(request) + parent = store.effective_spec(request["run_id"]) + chunk = ordered_chunks(json.loads(request["accepted_plan"]))[0] + child = worker_spec( + parent, + chunk, + {"url": "https://github.com/example/fixture/issues/10"}, + kind="chunk", + base_sha=parent["base_sha"], + base_branch="main", + ) + register_worker(store, parent, child) + broker = DeliveryBroker(store, child) + broker.prepare() + resources = RunResources(child) + stage = broker.checkout.with_name("staged-feature-candidate") + broker.checkout.rename(stage) + resources.register(broker.checkout, "checkout") + stage.rename(broker.checkout) + resources.created(broker.checkout) + (broker.checkout / "README.md").write_text("Retained feature candidate\n") + candidate = broker.candidate() + monkeypatch.setattr( + "devflow_temporal.delivery_preparation.verify_prepared_spec", lambda *_: None + ) + monkeypatch.setattr( + "devflow_temporal.delivery_configured_resources.implementation_prerequisites", lambda *_: [] + ) + monkeypatch.setattr( + "devflow_temporal.delivery_plan_checks.planned_checks", lambda *_a, **_k: [] + ) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda *_a, **_k: { + "state": "failed", + "results": [{"id": "missing", "passed": False, "cleanup": "confirmed"}], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + result = _check_readiness(broker, {"spec": child, "candidate": candidate, "iteration": 0}, 100) + implementation = { + "role": "implement", + "iteration": 0, + "status": "pass", + "session_id": "original-implementation", + "cleanup": "confirmed", + "candidate": candidate, + } + checks = {"prepublish": result, "readiness": result["readiness"]} + state = { + "run_id": child["run_id"], + "revision": 9, + "iteration": 0, + "phase": "blocked", + "execution_state": "blocked", + "outcome": "blocked", + "cleanup": "confirmed", + "candidate": candidate, + "pull_request": None, + "roles": [implementation], + "checks": checks, + "error": "execution readiness prevented prepublication", + } + store.project( + child["run_id"], + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=state["error"], + outcome="blocked", + cleanup="confirmed", + candidate=candidate, + checks=checks, + pull_request=None, + error=state["error"], + iteration=0, + protocol_revision=9, + ) + shared = registry(parent) + token = parent["feature_delivery"]["owner"] + shared.finish_worker(token, worker_key(child["run_id"], token), {"cleanup": "confirmed"}) + before = deepcopy(shared.budget(token["issue_id"])) + shared.stop(token, "stopped-for-preparation", {"cleanup": "confirmed"}) + with store._connect() as db: + store.state.release_work(db, parent["work_id"], "external:devflow:" + parent["run_id"]) + store.submit( + { + **request, + "run_id": "run-successor", + "command_id": "continue-preparation", + "feature_predecessor": token, + } + ) + successor = store.effective_spec("run-successor") + with store._connect() as db: + db.execute( + "INSERT INTO delivery_attempts(job_key,run_id,role,iteration,candidate_id," + "state,session_id,result_json,cleanup) VALUES " + "('implementation',?,'implement',0,?,'finished',?,?,'confirmed')", + ( + child["run_id"], + candidate["id"], + implementation["session_id"], + canonical_json( + { + k: v + for k, v in implementation.items() + if k not in {"role", "iteration", "candidate"} + } + ), + ), + ) + store.state.release_work(db, child["work_id"], "external:devflow:" + child["run_id"]) + row = dict( + db.execute("SELECT * FROM delivery_runs WHERE run_id=?", (child["run_id"],)).fetchone() + ) + closed = { + "workflow_id": row["workflow_id"], + "execution_run_id": "closed-worker", + "closed_at": "2020-01-01T00:00:00+00:00", + "request_digest": child["request_digest"], + "recovery_digest": None, + "result": state, + } + monkeypatch.setattr(store, "_completed_temporal_result", lambda *_a, **_k: closed) + monkeypatch.setattr(DeliveryBroker, "_existing_pr", lambda *_a, **_k: None) + monkeypatch.setattr( + "devflow_temporal.delivery_gate_retry._stopped_cleanup", lambda *_a, **_k: {} + ) + with pytest.raises(OwnershipConflict, match="active execution assignment"): + activities.resume_worker(store, successor, child, row) + assert shared.budget(token["issue_id"]) == before + with store._connect() as db: + assert not db.execute("SELECT 1 FROM delivery_gate_admissions").fetchone() + next_token = successor["feature_delivery"]["owner"] + shared.reserve_worker( + next_token, + worker_key(child["run_id"], next_token), + child["feature_worker"]["workstream_id"], + ) + continued = activities.resume_worker(store, successor, child, row) + assert continued["resumed"] and continued["spec"]["run_id"] == child["run_id"] + assert shared.budget(token["issue_id"]) == before + with store._connect() as db: + recovery = json.loads( + db.execute( + "SELECT recovery_json FROM delivery_runs WHERE run_id=?", (child["run_id"],) + ).fetchone()[0] + ) + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert recovery["command"]["additional_iterations"] == 0 + assert recovery["state"]["checks"]["prepublish"]["readiness"] == result["readiness"] diff --git a/runtime/tests/test_delivery_feature_revisions.py b/runtime/tests/test_delivery_feature_revisions.py index 04ba5eaf..d9aad92d 100644 --- a/runtime/tests/test_delivery_feature_revisions.py +++ b/runtime/tests/test_delivery_feature_revisions.py @@ -2,8 +2,10 @@ from __future__ import annotations +import asyncio import hashlib import json +import os import sys from copy import deepcopy from pathlib import Path @@ -157,7 +159,7 @@ def _revision_diagnostic_for(spec, diagnostic, candidate): def _native_rejected_fixture(native_configuration, monkeypatch, *, prior_cycles=4, - retain_worker=False): + retain_worker=False, early_prelaunch=False): config, request = native_configuration config.raw["max_repairs"] = 10 config.path.write_text(json.dumps(config.raw)) @@ -313,27 +315,49 @@ def observed_build(_store, run_id): **revisions.authenticate_revision_role(store, review), "proposed_plan": value, } - key, _ = supervisor._claim(review) - folder = Path(spec["state_dir"]) / "attempts" / key - review.update( - findings=[], - native_authorized=True, - role_evidence_key=key, - result_path=str(folder / "result.json"), - start_path=str(folder / "start.json"), - workspace=str(Path(spec["state_dir"]) / "gates" / "0" / "review"), - ) - review["review_diff"] = revision_diff(broker, review) - review["review_diff"].pop("candidate_id", None) - write_private(folder / "request.json", review) - with store._connect() as db: - expected = dict( - db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", (key,)).fetchone() + if early_prelaunch: + workspace = broker.gate_checkout("review", 0, candidate) + review["workspace"] = str(workspace) + review["review_diff"] = revision_diff(broker, review) + key = supervisor._job_key(review) + folder = Path(spec["state_dir"]) / "attempts" / key + + def refuse_preparation(*_args, **_kwargs): + raise ValueError("controlled verifier refusal before request persistence") + + def no_process(*_args, **_kwargs): + raise AssertionError("provider was unexpectedly allocated") + + with monkeypatch.context() as refusal: + refusal.setattr("devflow_temporal.delivery_preparation.verify_prepared_spec", + refuse_preparation) + refusal.setattr("devflow_temporal.delivery_native_process.NativeProcess", no_process) + failure = asyncio.run(supervisor.run(review)) + assert failure["finish_reason"] == "prelaunch" and failure["cleanup"] == "confirmed" + assert list(folder.iterdir()) == [] + else: + key, _ = supervisor._claim(review) + folder = Path(spec["state_dir"]) / "attempts" / key + review.update( + findings=[], + native_authorized=True, + role_evidence_key=key, + result_path=str(folder / "result.json"), + start_path=str(folder / "start.json"), + workspace=str(Path(spec["state_dir"]) / "gates" / "0" / "review"), ) - failure = supervisor._native_failure( - review, key, expected, "product patch shape rejected plan comparison", launch_absent=True - ) - assert failure["finish_reason"] == "prelaunch" and failure["cleanup"] == "confirmed" + review["review_diff"] = revision_diff(broker, review) + review["review_diff"].pop("candidate_id", None) + write_private(folder / "request.json", review) + with store._connect() as db: + expected = dict( + db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", (key,)).fetchone() + ) + failure = supervisor._native_failure( + review, key, expected, "product patch shape rejected plan comparison", + launch_absent=True, + ) + assert failure["finish_reason"] == "prelaunch" and failure["cleanup"] == "confirmed" rejection = revisions.reject_revision(store, spec, context["revision_id"], failure["summary"]) row = _stop_revision_fixture(store, spec, shared) return store, spec, shared, value, diagnostic, candidate, context, proposal, rejection, row @@ -578,3 +602,195 @@ def non_gate_drift(*args, **kwargs): revised_worker_spec) response = revisions.revise_feature_plan(store, original["run_id"], command) assert response["repair_budget"]["used"] == 6 + + +@pytest.mark.parametrize("early_prelaunch", [False, True]) +def test_public_revision_retains_unreviewed_prelaunch(native_configuration, monkeypatch, + early_prelaunch): + """A no-launch refusal is not a substantive rejection of a later correction.""" + import httpx + + from devflow_temporal.delivery_api import create_app + + native_configuration[0].raw["execution_mode"] = "trusted-local" + fixture = _native_rejected_fixture(native_configuration, monkeypatch, + early_prelaunch=early_prelaunch) + store, original, shared, plan, diagnostic, candidate, previous, prior_proposal, _, row = fixture + command = {"command_id": "prelaunch-revision", "expected_revision": row["revision"], + "reason": "Retry the correction after the environment refusal."} + + async def admit(): + app = create_app(store.config.path) + app.state.delivery.store = store + origin = store.config.dashboard_url.rstrip("/") + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), + base_url=origin) as client: + csrf = (await client.get("/api/session")).json()["csrf_token"] + headers = {"Origin": origin, "X-Devflow-CSRF": csrf} + url = f"/api/runs/{original['run_id']}/revise-feature-plan" + first = await client.post(url, json=command, headers=headers) + replay = await client.post(url, json=command, headers=headers) + assert first.status_code == 200, first.text + assert replay.status_code == 200 and replay.json() == first.json() + return first.json() + + admitted = asyncio.run(admit()) + successor = prepare_authority(store, store.effective_spec(admitted["run_id"])) + broker = DeliveryBroker(store, successor) + broker.prepare() + context = revisions.begin_revision(store, successor) + diagnostic = _revision_diagnostic_for(successor, diagnostic, broker.candidate()) + budget = shared.budget(previous["owner"]["issue_id"]) + assert budget["used"] == 6 + with store._connect() as db: + attempt = dict(db.execute( + "SELECT * FROM delivery_attempts WHERE run_id=? AND role='review'", + (original["run_id"],), + ).fetchone()) + folder = Path(attempt["result_path"]).parent + if early_prelaunch: + from concurrent.futures import ThreadPoolExecutor + from datetime import UTC, datetime + from threading import Event + + from devflow_temporal.delivery_resources import RunResources + + started = Event() + + def read_review(): + started.set() + return revisions._saved_revision_role( + store, original, previous, prior_proposal, candidate, "review", + closed_at=datetime(2099, 1, 1, tzinfo=UTC), + ) + + with ThreadPoolExecutor(max_workers=1) as executor: + registry = RunResources(original, read_only=True) + with registry.locked(read_only=True, exclusive=True): + pending = executor.submit(read_review) + assert started.wait(5) + with pytest.raises(TimeoutError): + pending.result(timeout=0.2) + review_result, review_proof = pending.result(timeout=15) + assert review_proof["classification"] == "prelaunch" + assert review_result["status"] == "blocked" + assert list(folder.iterdir()) == [] + + def refused(): + with pytest.raises(ValueError, + match="nonprogressing repeated proposal is already rejected"): + revisions.record_proposal(store, successor, context["revision_id"], plan, + diagnostic=diagnostic) + assert shared.budget(previous["owner"]["issue_id"]) == budget + + lease_path = Path(original["state_dir"]) / "attempt-owners" / attempt["job_key"] + assert lease_path.is_file() + lease = DeliverySupervisor._owner_lock(original, attempt["job_key"]) + assert lease is not None + try: + refused() + finally: + os.close(lease) + marker = folder / "launch.json" + marker.write_text("{}") + refused() + marker.unlink() + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET candidate_id=? WHERE job_key=?", + ("forged", attempt["job_key"])) + refused() + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET candidate_id=? WHERE job_key=?", + (attempt["candidate_id"], attempt["job_key"])) + adverse = {**json.loads(attempt["result_json"]), "status": "findings", + "finish_reason": "done"} + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET result_json=? WHERE job_key=?", + (canonical_json(adverse), attempt["job_key"])) + refused() + with store._connect() as db: + db.execute("UPDATE delivery_attempts SET result_json=? WHERE job_key=?", + (attempt["result_json"], attempt["job_key"])) + assert list(folder.iterdir()) == [] + proposal = revisions.record_proposal(store, successor, context["revision_id"], plan, + diagnostic=diagnostic) + assert proposal["unreviewed_predecessors"][0]["revision_id"] == previous["revision_id"] + assert shared.budget(previous["owner"]["issue_id"]) == budget + with store._connect() as db: + assert dict(db.execute("SELECT * FROM delivery_attempts WHERE job_key=?", + (attempt["job_key"],)).fetchone()) == attempt + +def test_public_reconsideration_preserves_early_and_genuine_reviews( + native_configuration, monkeypatch, tmp_path, +): + """An unlaunched review stays unreviewed beside a later genuine rejection.""" + import httpx + + from devflow_temporal.delivery_api import create_app + + native_configuration[0].raw['execution_mode'] = 'trusted-local' + fixture = _native_rejected_fixture(native_configuration, monkeypatch, + retain_worker='build', early_prelaunch=True) + store, original, shared, plan, diagnostic, _, prior, _, rejection, row = fixture + _, _, historical, current = context_repository(tmp_path, monkeypatch) + _complete_native_role_fixture(store, original, prior, plan, diagnostic, 'intake', historical) + command = {'command_id': 'mixed-ordinary-revision', 'expected_revision': row['revision'], + 'reason': 'Retry an unreviewed plan correction.'} + + async def post(run_id, command): + app = create_app(store.config.path) + app.state.delivery.store = store + origin = store.config.dashboard_url.rstrip('/') + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=origin) as c: + csrf = (await c.get('/api/session')).json()['csrf_token'] + return await c.post(f'/api/runs/{run_id}/revise-feature-plan', json=command, + headers={'Origin': origin, 'X-Devflow-CSRF': csrf}) + + early_selectors = {'version': 1, 'kind': 'derivation_context_change', + 'rejected_revision_id': prior['revision_id'], + 'proposal_sha256': digest(plan), + 'expected_runtime_revision': current['revision'], + 'expected_runtime_payload_sha256': current['runtime_payload_sha256']} + early_command = {**command, 'command_id': 'reject-unreviewed-reconsideration', + 'reconsideration': early_selectors} + denied = asyncio.run(post(original['run_id'], early_command)) + assert denied.status_code == 409, denied.text + assert 'genuine completed independent review' in denied.text + assert shared.budget(prior['owner']['issue_id'])['used'] == 5 + response = asyncio.run(post(original['run_id'], command)) + assert response.status_code == 200, response.text + successor = prepare_authority(store, store.effective_spec(response.json()['run_id'])) + broker = DeliveryBroker(store, successor) + broker.prepare() + with store._connect() as db: + db.execute('UPDATE delivery_runs SET candidate_json=? WHERE run_id=?', + (canonical_json(broker.candidate()), successor['run_id'])) + context = revisions.begin_revision(store, successor) + diagnostic = _revision_diagnostic_for(successor, diagnostic, broker.candidate()) + _complete_native_role_fixture(store, successor, context, plan, diagnostic, 'intake', historical) + proposal = revisions.record_proposal(store, successor, context['revision_id'], plan, + diagnostic=diagnostic) + assert proposal['unreviewed_predecessors'][0]['revision_id'] == prior['revision_id'] + _complete_native_role_fixture(store, successor, context, plan, diagnostic, 'review', historical) + rejected = revisions.reject_revision(store, successor, context['revision_id'], + 'A genuine independent review found a defect.') + stopped = _stop_revision_fixture(store, successor, shared) + selectors = {'version': 1, 'kind': 'derivation_context_change', + 'rejected_revision_id': context['revision_id'], 'proposal_sha256': digest(plan), + 'expected_runtime_revision': current['revision'], + 'expected_runtime_payload_sha256': current['runtime_payload_sha256']} + command = {'command_id':'mixed-reconsideration','expected_revision':stopped['revision'], + 'reason':'Reconsider after the verified derivation change.', + 'reconsideration':selectors} + response = asyncio.run(post(successor['run_id'], command)) + assert response.status_code == 200, response.text + proofs = response.json()['reconsideration']['prior_rejections'] + earlier = next(x for x in proofs if x['revision_id'] == prior['revision_id']) + later = next(x for x in proofs if x['revision_id'] == context['revision_id']) + assert earlier['classification'] == 'prelaunch' and len(earlier['review_intent_sha256']) == 64 + assert 'review_request_sha256' not in earlier + assert later['classification'] == 'genuine' and len(later['review_request_sha256']) == 64 + assert response.json()['repair_budget']['used'] == 6 + checkpoints = shared.checkpoints(prior['owner']['issue_id']) + assert checkpoints[revisions.PREFIX + 'rejected:' + prior['revision_id']] == rejection + assert checkpoints[revisions.PREFIX + 'rejected:' + context['revision_id']] == rejected diff --git a/runtime/tests/test_delivery_readiness_gate_retry.py b/runtime/tests/test_delivery_readiness_gate_retry.py new file mode 100644 index 00000000..9ee46d01 --- /dev/null +++ b/runtime/tests/test_delivery_readiness_gate_retry.py @@ -0,0 +1,921 @@ +"""Public same-candidate retry of a real, retained readiness refusal. + +The Temporal closure, implementation and failed dependency are controlled +fixtures. Readiness receipt creation, Git/SQLite ownership and HTTP admission +are real; no provider or production installation is used. +""" + +from __future__ import annotations + +import json +from contextlib import asynccontextmanager +from copy import deepcopy +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import httpx +import pytest +from test_delivery_gate_retry import unpublished as unpublished +from test_delivery_metadata_recovery import published as published +from test_delivery_store import _git +from test_delivery_store import service as base_service + +from devflow_temporal.contracts import canonical_json, digest +from devflow_temporal.delivery_activities import _check_readiness +from devflow_temporal.delivery_api import create_app +from devflow_temporal.delivery_broker import DeliveryBroker +from devflow_temporal.delivery_config import DeliveryConfig +from devflow_temporal.delivery_gate_retry import KIND, readback +from devflow_temporal.delivery_resources import RunResources +from devflow_temporal.delivery_store import DeliveryStore + + +@pytest.fixture +def service(tmp_path): + store, request = base_service.__wrapped__(tmp_path) + source = Path(store.config.raw["repositories"]["fixture"]["source_path"]) + (source / "test_retry_gate.py").write_text( + "import unittest\n" + "from pathlib import Path\n" + "class Gate(unittest.TestCase):\n" + " def test_candidate(self):\n" + " self.assertIn('Owned unpublished candidate', Path('README.md').read_text())\n" + " def test_absent(self):\n" + " self.assertIn('an absent requirement', Path('README.md').read_text())\n" + ) + _git(source, "add", "test_retry_gate.py") + _git(source, "commit", "-qm", "Add fixture check") + config = deepcopy(store.config.raw) + config["repositories"]["fixture"]["expected_base_sha"] = _git(source, "rev-parse", "HEAD") + store.config.path.write_text(json.dumps(config)) + return DeliveryStore(DeliveryConfig.load(store.config.path)), request + + +@pytest.fixture +def stopped_readiness(unpublished, monkeypatch, request): + store, _, state, command = unpublished + spec = deepcopy(store.spec("run-1")) + spec["readiness_version"] = 1 + spec["policy"]["prepublish_checks"] = [ + { + "id": "diff", + "argv": ["git", "diff", "--check"], + "cwd": ".", + "kind": "check", + "timeout_seconds": 30, + } + ] + selector = ( + "test_retry_gate.Gate.test_absent" + if getattr(request, "param", "") == "assertion" + else "test_retry_gate.Gate.test_candidate" + ) + spec["policy"]["checks"] = [ + { + "id": "assert-readme", + "argv": ["python3", "-m", "unittest", selector], + "cwd": ".", + "kind": "check", + "timeout_seconds": 30, + } + ] + spec["policy"]["readiness"] = { + "version": 1, + "generation": 0, + "browser_engines": [], + "temporal_checks": [], + } + browser_missing = getattr(request, "param", "") == "missing-browser" + if browser_missing: + spec["policy"]["browser_qa"] = {"argv": ["playwright", "test"], "ports": {}} + spec["policy"]["readiness"]["browser_engines"] = ["chromium"] + monkeypatch.setattr(RunResources, "browser_scratch", lambda self: Path(spec["state_dir"])) + monkeypatch.setattr( + "devflow_temporal.delivery_sandbox.prepare_browser_qa", + lambda *_args, **_kwargs: (None, {"PATH": str(Path(spec["state_dir"]) / "no-node")}), + ) + spec["policy_digest"] = digest(spec["policy"]) + with store._connect() as db: + db.execute( + "UPDATE delivery_runs SET request_json=? WHERE run_id=?", + (canonical_json(spec), "run-1"), + ) + broker = DeliveryBroker(store, spec) + resources = RunResources(spec) + staged = broker.checkout.with_name("staged-candidate") + broker.checkout.rename(staged) + resources.register(broker.checkout, "checkout") + staged.rename(broker.checkout) + resources.created(broker.checkout) + candidate = broker.candidate() + state["candidate"] = candidate + state["roles"][0]["candidate"] = candidate + with store._connect() as db: + role_result = { + k: v + for k, v in state["roles"][0].items() + if k not in {"role", "iteration", "candidate"} + } + db.execute( + "UPDATE delivery_attempts SET result_json=? WHERE job_key='original'", + (canonical_json(role_result),), + ) + command["expected_candidate_id"] = candidate["id"] + monkeypatch.setattr( + "devflow_temporal.delivery_preparation.verify_prepared_spec", lambda *_: None + ) + monkeypatch.setattr( + "devflow_temporal.delivery_configured_resources.implementation_prerequisites", lambda *_: [] + ) + monkeypatch.setattr( + "devflow_temporal.delivery_plan_checks.planned_checks", lambda *_a, **_k: [] + ) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, *_a, **_k: { + "state": "failed", + "results": [{"id": "dependency", "passed": False, "cleanup": "confirmed"}], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + + def failure(execution, current): + broker = DeliveryBroker(store, execution) + result = _check_readiness( + broker, {"spec": execution, "candidate": current["candidate"], "iteration": 0}, 100 + ) + assert result["failure_class"]["class"] == "environment_prelaunch" + assert result["readiness"]["state"] == "failed" + current["checks"] = {"prepublish": result, "readiness": result["readiness"]} + current["error"] = "execution readiness prevented prepublication" + return result + + original = failure(spec, state) + store.project( + "run-1", + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=state["error"], + candidate=candidate, + checks=state["checks"], + iteration=0, + protocol_revision=9, + outcome="blocked", + cleanup="confirmed", + error=state["error"], + ) + closed = { + "workflow_id": "delivery-run-1", + "execution_run_id": "closed-original", + "closed_at": "2020-01-01T00:00:00+00:00", + "request_digest": spec["request_digest"], + "recovery_digest": None, + "result": state, + } + monkeypatch.setattr(store, "_completed_temporal_result", lambda *_a, **_k: closed) + monkeypatch.setattr(DeliveryStore, "_completed_temporal_result", lambda *_a, **_k: closed) + return store, broker, state, command, original, closed, failure + + +@asynccontextmanager +async def client_for(store): + app = create_app(store.config.path) + host = store.config.dashboard_url + client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=app, client=("127.0.0.1", 8001)), base_url=host + ) + async with client: + headers = { + "Origin": host, + "X-Devflow-CSRF": (await client.get("/api/session")).json()["csrf_token"], + } + yield client, headers + + +@pytest.mark.asyncio +async def test_public_retry_preserves_receipts_and_bounds_repeated_failures( + stopped_readiness, monkeypatch +): + store, broker, state, command, original, closed, failure = stopped_readiness + from devflow_temporal import delivery_readiness_gate_retry + + clock = [datetime(2020, 1, 1, tzinfo=UTC)] + monkeypatch.setattr(delivery_readiness_gate_retry, "_utc_now", lambda: clock[0], raising=False) + closed["closed_at"] = clock[0].isoformat() + candidate = deepcopy(state["candidate"]) + retained = Path(original["readiness"]["receipt"]).read_bytes() + with store._connect() as db: + initial_attempts = [dict(x) for x in db.execute("SELECT * FROM delivery_attempts")] + initial_effects = [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] + attempts = [original["readiness"]["attempt"]] + async with client_for(store) as (client, headers): + # A missing close time cannot grant a retry or silently disable backoff. + for invalid_close in (None, "2020-01-01T00:00:00"): + closed["closed_at"] = invalid_close + assessment = await client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert assessment.status_code == 200 + assert any(item["code"] == "readiness_retry_clock_unknown" + and item["satisfied"] is None + for item in assessment.json()["blockers"]) + closed["closed_at"] = clock[0].isoformat() + for generation in (1, 2): + # Infrastructure retries wait 30 seconds and then 60 seconds; + # no product allowance or attempt is allocated during the wait. + waiting = await client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert waiting.status_code == 200 + assert any(item["code"] == "readiness_retry_backoff" + for item in waiting.json()["blockers"]) + if generation == 2: + # Restarting the control service must not reset the delay. + restarted_store = DeliveryStore(DeliveryConfig.load(store.config.path)) + async with client_for(restarted_store) as (restarted_client, _): + after_restart = await restarted_client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert after_restart.status_code == 200 + assert any(item["code"] == "readiness_retry_backoff" + for item in after_restart.json()["blockers"]) + with store._connect() as db: + admissions = [dict(x) for x in db.execute("SELECT * FROM delivery_gate_admissions")] + denied = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert denied.status_code == 409 + with store._connect() as db: + assert [dict(x) for x in db.execute("SELECT * FROM delivery_attempts")] == ( + initial_attempts + ) + assert [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] == ( + initial_effects + ) + assert [dict(x) for x in db.execute("SELECT * FROM delivery_gate_admissions")] == ( + admissions + ) + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + delay = 30 * 2 ** (generation - 1) + clock[0] += timedelta(seconds=delay - 1) + before_deadline = await client.post( + "/api/runs/run-1/repair-admission-preflight", json=command + ) + assert before_deadline.status_code == 409 + assert "readiness_retry_backoff" in before_deadline.text + clock[0] += timedelta(seconds=1) + preflight = await client.post( + "/api/runs/run-1/repair-admission-preflight", json=command + ) + assert preflight.status_code == 200, preflight.text + result = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert result.status_code == 200, result.text + admitted = result.json() + assert admitted["implementation_authority"] is False + assert ( + admitted["additional_iterations"] == 0 + and admitted["candidate_id"] == candidate["id"] + ) + assert ( + await client.post("/api/runs/run-1/continue-repair", json=command, headers=headers) + ).json() == admitted + execution = store.effective_spec("run-1") + with store._connect() as db: + recovery = json.loads( + db.execute( + "SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'" + ).fetchone()[0] + ) + assert [ + dict(x) for x in db.execute("SELECT * FROM delivery_attempts") + ] == initial_attempts + assert [ + dict(x) for x in db.execute("SELECT * FROM delivery_effects") + ] == initial_effects + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert execution["gate_retry_generation"] == generation + assert execution["gate_retry_stage"] == "readiness" + assert recovery["seal"]["readiness_backoff"]["delay_seconds"] == delay + assert recovery["seal"]["readiness_backoff"]["retry_not_before"] == ( + clock[0].isoformat() + ) + assert recovery["state"]["checks"]["prepublish"]["readiness"]["state"] == "failed" + assert readback(store, execution, recovery) is None + if generation == 1: + old_path = Path(original["readiness"]["receipt"]) + old_path.write_bytes(retained + b" ") + with pytest.raises(ValueError, match="receipt changed"): + readback(store, execution, recovery) + old_path.write_bytes(retained) + assert broker.candidate() == candidate + current = deepcopy(recovery["state"]) + current.update( + revision=9 + generation, + phase="blocked", + execution_state="blocked", + outcome="blocked", + cleanup="confirmed", + candidate=recovery["candidate"], + ) + failed_again = failure(execution, current) + attempts.append(failed_again["readiness"]["attempt"]) + store.project( + "run-1", + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=current["error"], + candidate=current["candidate"], + checks=current["checks"], + iteration=0, + protocol_revision=current["revision"], + outcome="blocked", + cleanup="confirmed", + error=current["error"], + ) + with store._connect() as db: + store.state.release_work(db, execution["work_id"], "external:devflow:run-1") + closed.update( + workflow_id=admitted["workflow_id"], + execution_run_id=f"closed-{generation}", + closed_at=clock[0].isoformat(), + recovery_digest=digest(recovery), + result=current, + ) + command = { + **command, + "command_id": f"retry-readiness-{generation + 1}", + "expected_revision": current["revision"], + } + refusal = await client.post("/api/runs/run-1/repair-admission-preflight", json=command) + assert refusal.status_code == 409 + assert "readiness_retry_limit_exhausted" in refusal.text + assert len(set(attempts)) == 3 + assert Path(original["readiness"]["receipt"]).read_bytes() == retained + with store._connect() as db: + assert [dict(x) for x in db.execute("SELECT * FROM delivery_attempts")] == initial_attempts + assert [dict(x) for x in db.execute("SELECT * FROM delivery_effects")] == initial_effects + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drift", ["receipt", "source", "effect", "product", "cleanup"]) +async def test_public_retry_refuses_changed_or_unsettled_evidence(stopped_readiness, drift): + store, broker, state, command, original, _, _ = stopped_readiness + if drift == "receipt": + path = Path(original["readiness"]["receipt"]) + path.write_bytes(path.read_bytes() + b" ") + elif drift == "source": + (broker.checkout / "README.md").write_text("Changed candidate\n") + elif drift == "effect": + with store._connect() as db: + db.execute( + "INSERT INTO delivery_effects VALUES ('pending','run-1','publish'," + "'{}','pending',NULL,'now')" + ) + elif drift == "product": + state["roles"].append( + { + "role": "review", + "iteration": 0, + "status": "findings", + "findings": ["Real product defect"], + "cleanup": "confirmed", + } + ) + else: + state["cleanup"] = "unknown" + with store._connect() as db: + db.execute("UPDATE delivery_runs SET cleanup='unknown' WHERE run_id='run-1'") + async with client_for(store) as (client, _): + response = await client.post("/api/runs/run-1/repair-admission-preflight", json=command) + assert response.status_code == 409, response.text + with store._connect() as db: + assert not db.execute("SELECT 1 FROM delivery_gate_admissions").fetchone() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stopped_readiness", ["passing", "assertion"], indirect=True) +async def test_retry_workflow_runs_the_frozen_gates_without_implementation( + stopped_readiness, monkeypatch +): + import asyncio + + from temporal_test_server import local_temporal + from temporalio import activity + from temporalio.worker import Worker + + from devflow_temporal.delivery_activities import ( + delivery_checks, + delivery_gates_readback, + delivery_precheck, + delivery_project, + ) + from devflow_temporal.delivery_workflow import DeliveryWorkflow + + store, broker, state, command, original, _, _ = stopped_readiness + old_receipt = Path(original["readiness"]["receipt"]).read_bytes() + async with client_for(store) as (client, headers): + response = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert response.status_code == 200, response.text + admitted = response.json() + spec = store.effective_spec("run-1") + with store._connect() as db: + recovery = json.loads( + db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0] + ) + store.mark_start("run-1", accepted=True) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, _i, candidate, **_k: { + "state": "passed", + "results": [], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + monkeypatch.setattr( + "devflow_temporal.delivery_readiness.probe_tools", + lambda _broker, requirements, candidate, **_kw: { + "state": "passed", + "cleanup": "confirmed", + "component": "tools", + "candidate": candidate, + "requirement_sha256": digest(requirements), + "items": [], + }, + ) + monkeypatch.setattr("devflow_temporal.delivery_readiness_verify.tools", lambda *_a: None) + actual_gate_checkout = DeliveryBroker.gate_checkout + + def gate_checkout(self, role, iteration, candidate): + path = actual_gate_checkout(self, role, iteration, candidate) + resources = self.run_resources() + with resources.locked(read_only=True) as manifest: + registered = str(path) in manifest["roots"] + if not registered: + staged = path.with_name(path.name + "-staged") + path.rename(staged) + resources.register(path, "gate") + staged.rename(path) + resources.created(path, head=candidate["head"]) + return path + + monkeypatch.setattr(DeliveryBroker, "gate_checkout", gate_checkout) + calls = [] + + @activity.defn(name="delivery_publish") + async def publish(body): + calls.append("publish") + assert body["candidate"]["id"] == state["candidate"]["id"] + _git(broker.checkout, "add", "README.md") + _git(broker.checkout, "commit", "-qm", "Publish fixture") + current = DeliveryBroker(store, body["spec"]).candidate() + return { + "number": 7, + "url": "https://example.invalid/pull/7", + "state": "OPEN", + "head": current["head"], + "candidate": current, + } + + @activity.defn(name="delivery_role") + async def role(body): + calls.append(body["role"]) + assert body["role"] in {"review", "verify"} + return { + "role": body["role"], + "iteration": body["iteration"], + "status": "pass", + "session_id": "independent-" + body["role"], + "candidate": body["candidate"], + "cleanup": "confirmed", + "findings": [], + } + + @activity.defn(name="delivery_tracker_start") + async def start(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_tracker") + async def tracker(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_ci") + async def ci(_body): + calls.append("ci") + return {"state": "passed"} + + async with local_temporal() as env: + async with Worker( + env.client, + task_queue="issue-243-gates", + workflows=[DeliveryWorkflow], + activities=[ + delivery_gates_readback, + delivery_project, + delivery_precheck, + delivery_checks, + publish, + role, + start, + tracker, + ci, + ], + ): + handle = await env.client.start_workflow( + DeliveryWorkflow.run, + args=[spec, recovery], + id=admitted["workflow_id"], + task_queue="issue-243-gates", + ) + result = await asyncio.wait_for(handle.result(), 60) + if spec["policy"]["checks"][0]["argv"][-1].endswith("test_absent"): + assert result["outcome"] == "blocked" + assert result["checks"]["local"]["state"] == "failed" + assert result["checks"]["local"]["results"][0]["exit_code"] != 0 + assert any("local_checks" in finding for finding in result["findings"]) + assert calls == ["publish"] + else: + assert result["outcome"] == "delivered", result + assert calls == ["publish", "review", "verify", "ci"] + assert result["iteration"] == 0 + assert result["roles"][0] == state["roles"][0] + assert ( + result["checks"]["prepublish"]["results"][0]["argv"] + == spec["policy"]["prepublish_checks"][0]["argv"] + ) + assert result["checks"]["local"]["results"][0]["argv"] == spec["policy"]["checks"][0]["argv"] + assert recovery["original_spec"]["policy"]["checks"] == spec["policy"]["checks"] + assert Path(original["readiness"]["receipt"]).read_bytes() == old_receipt + with store._connect() as db: + assert db.execute("SELECT COUNT(*) FROM delivery_attempts").fetchone()[0] == 1 + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + + +@pytest.mark.asyncio +async def test_public_native_retry_reports_missing_descendant_settlement(stopped_readiness): + """A plausible native-shaped checkpoint still cannot bypass missing settlement proof.""" + import hashlib + + from devflow_temporal.delivery_readiness_gate_retry import failure as classify + from devflow_temporal.delivery_readiness_gate_retry import probe_attempt + from devflow_temporal.delivery_resources import write_private + + store, _, state, command, result, closed, _ = stopped_readiness + spec = deepcopy(store.spec("run-1")) + spec["provider"] = "codex" + spec["policy"]["host_sandbox"] = "trusted-local" + spec["policy"]["execution_backend"] = "native-macos" + spec["policy_digest"] = digest(spec["policy"]) + candidate = deepcopy(state["candidate"]) + candidate["policy_digest"] = spec["policy_digest"] + state["candidate"] = candidate + state["roles"][0]["candidate"] = candidate + receipt = result["readiness"] + receipt["binding"]["spec_sha256"] = digest(spec) + receipt["binding"]["candidate"] = candidate + for observation in receipt["observations"].values(): + observation["candidate"] = candidate + receipt["attempt"] = probe_attempt(spec, candidate, 100) + path = ( + Path(spec["state_dir"]) + / "readiness" + / "receipts" + / "plan" + / "0" + / f"{receipt['attempt']}.json" + ) + receipt["receipt"] = str(path) + write_private( + path, + {key: value for key, value in receipt.items() if key not in {"receipt", "receipt_sha256"}}, + ) + receipt["receipt_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + result["candidate_id"] = candidate["id"] + result["failure_class"] = classify(receipt, "prepublish", candidate["id"], "confirmed") + with store._connect() as db: + role_result = { + k: v + for k, v in state["roles"][0].items() + if k not in {"role", "iteration", "candidate"} + } + db.execute( + "UPDATE delivery_attempts SET result_json=? WHERE job_key='original'", + (canonical_json(role_result),), + ) + db.execute( + "UPDATE delivery_runs SET request_json=?,candidate_json=?,checks_json=? " + "WHERE run_id='run-1'", + (canonical_json(spec), canonical_json(candidate), canonical_json(state["checks"])), + ) + closed["result"] = state + async with client_for(store) as (client, headers): + assessment = await client.post( + "/api/runs/run-1/assess-transition", + json={"kind": command["continuation_kind"], "request": command}, + ) + assert assessment.status_code == 200, assessment.text + data = assessment.json() + assert data["eligible"] is False + assert any( + item["code"] == "native_descendant_settlement_unknown" and item["satisfied"] is None + for item in data["blockers"] + ), data + denied = await client.post("/api/runs/run-1/continue-repair", json=command, headers=headers) + assert denied.status_code == 409, denied.text + assert any( + item["code"] == "native_descendant_settlement_unknown" + for item in denied.json()["detail"]["blockers"] + ) + with store._connect() as db: + assert not db.execute("SELECT 1 FROM delivery_gate_admissions").fetchone() + + +@pytest.mark.asyncio +async def test_public_published_readiness_retains_candidate_and_full_selector( + published, monkeypatch +): + store, _, state, closed, _, _ = published + spec = deepcopy(store.spec("run-1")) + spec["readiness_version"] = 1 + spec["resource_cleanup_version"] = 1 + spec["policy"]["readiness"] = { + "version": 1, + "generation": 0, + "browser_engines": [], + "temporal_checks": [], + } + spec["policy_digest"] = digest(spec["policy"]) + with store._connect() as db: + db.execute( + "UPDATE delivery_runs SET request_json=? WHERE run_id=?", + (canonical_json(spec), "run-1"), + ) + broker = DeliveryBroker(store, spec) + resources = RunResources(spec) + staged = broker.checkout.with_name("staged-published-candidate") + broker.checkout.rename(staged) + resources.register(broker.checkout, "checkout") + staged.rename(broker.checkout) + resources.created(broker.checkout) + candidate = broker.candidate() + state["candidate"] = candidate + state["pull_request"]["candidate"] = candidate + state["roles"][-1]["candidate"]["policy_digest"] = spec["policy_digest"] + with store._connect() as db: + role_result = { + k: v + for k, v in state["roles"][-1].items() + if k not in {"role", "iteration", "candidate", "input_candidate_id"} + } + db.execute( + "UPDATE delivery_attempts SET result_json=? WHERE job_key='implementation-1'", + (canonical_json(role_result),), + ) + store.state.release_work(db, spec["work_id"], "external:devflow:run-1") + monkeypatch.setattr( + "devflow_temporal.delivery_preparation.verify_prepared_spec", lambda *_: None + ) + monkeypatch.setattr( + "devflow_temporal.delivery_configured_resources.implementation_prerequisites", lambda *_: [] + ) + monkeypatch.setattr( + "devflow_temporal.delivery_plan_checks.planned_checks", lambda *_a, **_k: [] + ) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, *_a, **_k: { + "state": "failed", + "results": [{"id": "dependency", "passed": False, "cleanup": "confirmed"}], + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + ) + readiness = _check_readiness( + broker, {"spec": spec, "candidate": candidate, "iteration": 1}, 101 + ) + assert readiness["failure_class"]["class"] == "environment_prelaunch" + checks = { + "prepublish": { + "state": "passed", + "candidate_id": candidate["id"], + "source_unchanged": True, + }, + "local": readiness, + "readiness": readiness["readiness"], + } + state.update( + checks=checks, cleanup="confirmed", error="execution readiness prevented local checks" + ) + store.project( + "run-1", + phase="blocked", + execution_state="blocked", + event_type="blocked", + message=state["error"], + candidate=candidate, + pull_request=state["pull_request"], + checks=checks, + iteration=1, + protocol_revision=13, + outcome="blocked", + cleanup="confirmed", + error=state["error"], + ) + closed["result"] = state + monkeypatch.setattr(DeliveryStore, "_completed_temporal_result", lambda *_a, **_k: closed) + monkeypatch.setattr("devflow_temporal.delivery_gate_retry._stopped_cleanup", lambda *a, **k: {}) + monkeypatch.setattr("devflow_temporal.delivery_gate_retry.preserve_resources", lambda *_: None) + command = { + "continuation_kind": KIND, + "command_id": "published-readiness-retry", + "expected_revision": 13, + "expected_iteration": 1, + "expected_candidate_id": candidate["id"], + "expected_pr_number": 7, + "expected_pr_head": candidate["head"], + "additional_iterations": 0, + } + with store._connect() as db: + effects = [ + dict(row) for row in db.execute("SELECT * FROM delivery_effects ORDER BY effect_key") + ] + attempts = [ + dict(row) for row in db.execute("SELECT * FROM delivery_attempts ORDER BY job_key") + ] + async with client_for(store) as (client, headers): + narrower = await client.post( + "/api/runs/run-1/repair-admission-preflight", + json={**command, "verification_test_paths": ["test_retry_gate.py"]}, + ) + assert narrower.status_code == 409 and "frozen test selectors" in narrower.text + preflight = await client.post("/api/runs/run-1/repair-admission-preflight", json=command) + assert preflight.status_code == 200, preflight.text + response = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert response.status_code == 200, response.text + assert response.json()["implementation_authority"] is False + execution = store.effective_spec("run-1") + with store._connect() as db: + recovery = json.loads( + db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0] + ) + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_effects ORDER BY effect_key") + ] == effects + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_attempts ORDER BY job_key") + ] == attempts + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert recovery["publication"] == state["pull_request"] + assert recovery["state"]["checks"]["local"] == readiness + assert execution["policy"]["checks"] == spec["policy"]["checks"] + assert readback(store, execution, recovery) == state["pull_request"] + + import asyncio + + from temporal_test_server import local_temporal + from temporalio import activity + from temporalio.worker import Worker + + from devflow_temporal.delivery_activities import ( + delivery_checks, + delivery_finalize_resources, + delivery_gates_readback, + delivery_precheck, + delivery_project, + ) + from devflow_temporal.delivery_workflow import DeliveryWorkflow + + store.mark_start("run-1", accepted=True) + monkeypatch.setattr( + DeliveryBroker, + "run_implementation_preparation", + lambda self, _i, value, **_k: { + "state": "passed", + "results": [], + "candidate_id": value["id"], + "source_unchanged": True, + }, + ) + monkeypatch.setattr( + "devflow_temporal.delivery_readiness.probe_tools", + lambda _broker, requirements, value, **_kw: { + "state": "passed", + "cleanup": "confirmed", + "component": "tools", + "candidate": value, + "requirement_sha256": digest(requirements), + "items": [], + }, + ) + monkeypatch.setattr("devflow_temporal.delivery_readiness_verify.tools", lambda *_a: None) + calls = [] + + @activity.defn(name="delivery_publish") + async def unexpected_publish(_body): + pytest.fail("published retry must never publish the candidate again") + + @activity.defn(name="delivery_role") + async def role(body): + calls.append(body["role"]) + assert body["role"] in {"review", "verify"} + assert body["candidate"] == candidate + return { + "role": body["role"], + "iteration": body["iteration"], + "status": "pass", + "session_id": "independent-" + body["role"], + "candidate": body["candidate"], + "cleanup": "confirmed", + "findings": [], + } + + @activity.defn(name="delivery_tracker_start") + async def start(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_tracker") + async def tracker(_body): + return {"state": "consistent"} + + @activity.defn(name="delivery_ci") + async def ci(body): + calls.append("ci") + assert body["pull_request"] == state["pull_request"] + return {"state": "passed"} + + async with local_temporal() as env: + async with Worker( + env.client, + task_queue="issue-243-published", + workflows=[DeliveryWorkflow], + activities=[ + delivery_gates_readback, + delivery_project, + delivery_precheck, + delivery_checks, + delivery_finalize_resources, + unexpected_publish, + role, + start, + tracker, + ci, + ], + ): + handle = await env.client.start_workflow( + DeliveryWorkflow.run, + args=[execution, recovery], + id=response.json()["workflow_id"], + task_queue="issue-243-published", + ) + finished = await asyncio.wait_for(handle.result(), 60) + assert finished["outcome"] == "delivered", finished + assert calls == ["review", "verify", "ci"] + assert finished["candidate"] == candidate + assert finished["checks"]["local"]["results"][0]["argv"] == spec["policy"]["checks"][0]["argv"] + with store._connect() as db: + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_effects ORDER BY effect_key") + ] == effects + assert [ + dict(row) for row in db.execute("SELECT * FROM delivery_attempts ORDER BY job_key") + ] == attempts + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stopped_readiness", ["missing-browser"], indirect=True) +async def test_public_retry_retains_missing_browser_runtime_observation(stopped_readiness): + store, _, state, command, original, _, _ = stopped_readiness + observation = original["readiness"]["observations"]["browser"] + assert observation["state"] == "failed" and observation["launched"] is False + assert observation["reason"] == "node unavailable in browser environment" + async with client_for(store) as (client, headers): + response = await client.post( + "/api/runs/run-1/continue-repair", json=command, headers=headers + ) + assert response.status_code == 200, response.text + assert response.json()["implementation_authority"] is False + with store._connect() as db: + recovery = json.loads( + db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0] + ) + assert not db.execute("SELECT 1 FROM delivery_repair_grants").fetchone() + assert db.execute("SELECT COUNT(*) FROM delivery_attempts").fetchone()[0] == 1 + assert recovery["state"]["checks"]["prepublish"]["readiness"] == original["readiness"] + assert recovery["candidate"] == state["candidate"]