diff --git a/docs/temporal-runtime.md b/docs/temporal-runtime.md index 242aac20..acf5b51c 100644 --- a/docs/temporal-runtime.md +++ b/docs/temporal-runtime.md @@ -238,6 +238,48 @@ For an ordinary published run stopped with `browser QA child cleanup is unknown` For an ordinary published run whose native roles have finished but whose resource cleanup remains unknown, `reconcile-attempt --id RUN --role cleanup --iteration N` authenticates the exact closed workflow and re-observes its native resources before closing its claim. An unpublished run is also eligible after it has an accepted plan and an exact completed implementation/source checkpoint. Its closure is recorded as `unpublished`, with the closed execution identity, original checks/results, current source and work binding, attempt/effect inventory, native observation and tracker acknowledgement. It independently observes the configured origin and absence of the owned remote branch/PR before finalization and again before claim release. Pre-intake runs, a newer issue owner, changed work bindings, uncertain effects or live native custody prevent closure. Repeating the command re-observes its exact acknowledged custody without starting another role, admitting a successor or rewriting the historical verdict. Dirty retained source, publication state, failure classification and original repair allowance remain intact. Workspace dependency cleanup uses the declaration recorded when the runtime acquired custody, so permitted package script edits do not invalidate deletion of those same owned folders; replacement folders and symlinks remain protected. +Recovery callers can inspect a specific request before admission using +`assess-transition --id RUN --request FILE`; the JSON file contains `kind` and the +corresponding recovery `request`. The API accepts the same object at +`POST /api/runs/RUN/assess-transition`. Version 1 returns a lifecycle observation, +an advisory transition for supported routes, candidate and revision observations, +and structured blockers. A blocker identifies its owner, the missing or unknown +fact, a next action, and whether new authorization or user input is required. +`user_input_required: null` means that need has not been established; it is not a +request for approval. `satisfied: null` means the fact is unknown, while `false` +means the observed predicate did not hold. Some older owner adapters still return +a general unknown when they cannot authenticate a more specific fact. + +The lifecycle vocabulary distinguishes pending, running, stopping, stopped, +recovering, terminal and unknown projections. A projection or successful +assessment is not a custody grant: the owning admission rechecks the exact +candidate, original authority, effects, cleanup and finite budget under its +existing fences. Historical ordinary repair also checks the original work, issue +and repository binding in both preview and admission. The transition table does +not choose a route. Pause states are +reserved for the pause owner; queued pause support does not prove a running +workflow has settled. An ordinary preparation failure before plan acceptance +cannot use implementation resume without an accepted plan. Inspect the retained +intake and original custody; do not treat a missing plan or unknown cleanup as a +completed implementation or a released owner. + +When an exact ordinary repair, stopped-resume, gate-retry, or pending-publication +command has already been admitted, assessment reports `already_admitted` rather +than presenting +the old receipt as a fresh grant. Replaying the same command returns its original +receipt; reusing its ID with different inputs reports `command_conflict` and queues +no new work. A missing required receipt field, mismatched workflow, or unsupported +retained admission is unknown. Workflow checks use retained operation evidence and +do not depend on the current workflow, so later progress does not invalidate an +authentic replay. Historical repair and stopped-resume events did not independently +seal every presentation field, including the dashboard URL; those fields can be +checked for required shape, but their original bytes cannot always be reconstructed. +The stopped-resume receipt check verifies the form of its admission hash; its owner +separately authenticates the retained admission during execution custody checks. +Pending publication also reports unknown native resource custody when the original +process, root, or lease cannot be proven settled; the existing observer still refuses +admission. + After that closure, use the public `repair-admission-preflight --id RUN --request FILE` and `continue-repair --id RUN --request FILE` route with `continuation_kind: "stopped_delivery_resume"`, a stable unique `command_id`, the closed protocol `expected_revision`, `expected_iteration`, and current `expected_candidate_id`/`expected_candidate_head`. A passed implementation stopped with exact unknown prepublication cleanup resumes at the sealed iteration's checks with `additional_iterations: 0`. Its exact durable implementation result and unchanged candidate remain mandatory; all publication, independent review, QA, verification, required CI and tracker gates still run. A provider monitor lost after launch remains an unknown verdict. For the narrow original unpublished implementation checkpoint, public cleanup can independently authenticate finished teardown without completing monitoring or inventing a provider session. The closed role, durable supervisor result, exact private request/journal hashes, input candidate, current partial output, provider identity and unknown outcome are bound separately. A later explicit `stopped_delivery_resume` must grant one or two unused original iterations (`additional_iterations: 1` or `2`), starts at the next iteration in a fresh implementation home/session, and receives no invented model finding from the old infrastructure failure. The old result, role and monitoring state remain unchanged. Subsequent normal resumes retain only those exact historical teardown receipts through the immutable admission chain; a different unknown role, changed output/evidence or current live custody still blocks them. Unknown external effects are never replayed. diff --git a/runtime/src/devflow_temporal/delivery_gate_retry.py b/runtime/src/devflow_temporal/delivery_gate_retry.py index a68a766e..10d2260c 100644 --- a/runtime/src/devflow_temporal/delivery_gate_retry.py +++ b/runtime/src/devflow_temporal/delivery_gate_retry.py @@ -766,46 +766,85 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= raise ValueError('gate retry requires the existing trusted native execution policy') unpublished_error = (state.get('error') if prepublication_preparation_failed(state) else 'prepublication repair limit exhausted') - if ((kind == PRELAUNCH_KIND and not prelaunch) - or (previous is not None and not ( - renewed or published_after_recovery or prelaunch - or report_retry or ci_only or transient_retry or attribution_retry - or verification_retry or browser_artifact_retry or prerequisite_retry)) - or row['phase'] != 'blocked' or row['outcome'] != 'blocked' - or row['execution_state'] != 'blocked' - or row['cleanup'] != ('unknown' if prelaunch else 'confirmed') - or state.get('phase') != 'blocked' or state.get('outcome') != 'blocked' - or (prelaunch and state.get('cleanup') != 'unknown') - or (not prelaunch and state.get('cleanup') != 'confirmed' and reconciled is None) - or row['error'] != state.get('error') - or (not (transient_retry or transient_ci or masked_exhaustion) - and state.get('error') not in ({'repair limit exhausted', - 'required CI did not confirm this PR head'} - if report_retry - else {'required CI did not confirm this PR head' - if ci_only - else 'browser QA child cleanup is unknown' - if browser_prelaunch else - 'local check process cleanup is unknown' if prelaunch - else unpublished_error - if unpublished else state.get('error') - if (controller_only or attribution_retry - or browser_artifact_retry or prerequisite_retry) - else 'repair limit exhausted'})) - or row['protocol_revision'] != state.get('revision') - or row['iteration'] != state.get('iteration') - or closed['request_digest'] != row['request_digest'] - or closed['recovery_digest'] != (digest(previous) if previous else None) - or not implementation or (implementation.get('status') != 'pass' - and not (controller_only or attribution_retry - or browser_artifact_retry - or layout_report or verification_retry)) - or not failed_gate - or any(a['state'] != 'finished' or a['cleanup'] != 'confirmed' for a in attempts) - or any((e['state'] != 'complete' or not e['observed_json']) - and e != (browser_observation or {}).get('pending_effect') for e in effects) - or (claim is not None and not prelaunch)): - raise ValueError('gate retry requires a closed, finalized failed gate and released claim') + # Report the original guard's independent facts through the shared + # evaluator. The same snapshot is used for observation and + # admission, so neither surface needs to interpret English error strings. + from .delivery_lifecycle import lifecycle_from_row, require_observed_predicates + + checkpoint_matches = not ( + (kind == PRELAUNCH_KIND and not prelaunch) + or (previous is not None and not ( + renewed or published_after_recovery or prelaunch or report_retry or ci_only + or transient_retry or attribution_retry or verification_retry + or browser_artifact_retry or prerequisite_retry)) + or row['phase'] != 'blocked' or row['outcome'] != 'blocked' + or row['execution_state'] != 'blocked' + or state.get('phase') != 'blocked' or state.get('outcome') != 'blocked' + or row['error'] != state.get('error') + or row['protocol_revision'] != state.get('revision') + or row['iteration'] != state.get('iteration') + or closed['request_digest'] != row['request_digest'] + or closed['recovery_digest'] != (digest(previous) if previous else None)) + cleanup_matches = (row['cleanup'] == ('unknown' if prelaunch else 'confirmed') + and ((state.get('cleanup') == 'unknown') if prelaunch else + (state.get('cleanup') == 'confirmed' or reconciled is not None))) + verdict_matches = ( + (transient_retry or transient_ci or masked_exhaustion + or state.get('error') in ({'repair limit exhausted', + 'required CI did not confirm this PR head'} if report_retry + else {'required CI did not confirm this PR head' if ci_only + else 'browser QA child cleanup is unknown' if browser_prelaunch + else 'local check process cleanup is unknown' if prelaunch + else unpublished_error if unpublished + else state.get('error') if (controller_only or attribution_retry + or browser_artifact_retry or prerequisite_retry) + else 'repair limit exhausted'})) + and bool(implementation) + and (implementation.get('status') == 'pass' or controller_only or attribution_retry + or browser_artifact_retry or layout_report or verification_retry) + and failed_gate) + frozen_candidate = json.loads(row['candidate_json'] or 'null') + candidate_matches = (isinstance(frozen_candidate, dict) + and not any(candidate.get(k) != frozen_candidate.get(k) for k in candidate) + and frozen_candidate == state.get('candidate')) + attempts_settled = all(a['state'] == 'finished' and a['cleanup'] == 'confirmed' + for a in attempts) + effects_settled = all((e['state'] == 'complete' and bool(e['observed_json'])) + or e == (browser_observation or {}).get('pending_effect') + for e in effects) + released = claim is None or prelaunch + require_observed_predicates(kind, 'delivery_gate_retry', {'row': row, 'closed': closed}, [ + {'code': 'gate_checkpoint_unproven', 'satisfied': checkpoint_matches, + 'missing_or_unknown_fact': 'matching stopped run and closed predecessor', + 'next_action': 'inspect the closed run and its retained recovery history', + 'user_input_required': None}, + {'code': 'gate_cleanup_unconfirmed', 'satisfied': True if cleanup_matches else None, + 'missing_or_unknown_fact': 'confirmed cleanup or authenticated later reconciliation', + 'next_action': 'reconcile the original cleanup and custody', + 'user_input_required': None}, + {'code': 'gate_candidate_changed', 'satisfied': candidate_matches, + 'missing_or_unknown_fact': 'unchanged candidate matching the closed checkpoint', + 'next_action': 'inspect the retained candidate and obtain fresh owner proof', + 'user_input_required': None}, + {'code': 'failed_gate_unproven', 'satisfied': verdict_matches, + 'missing_or_unknown_fact': 'failed gate and its original implementation verdict', + 'next_action': 'inspect the retained gate and implementation evidence', + 'user_input_required': None}, + {'code': 'gate_attempts_unsettled', 'satisfied': True if attempts_settled else None, + 'missing_or_unknown_fact': 'finished attempts and confirmed cleanup', + 'next_action': 'reconcile the original attempt custody', + 'user_input_required': None}, + {'code': 'gate_effects_unsettled', 'satisfied': True if effects_settled else None, + 'missing_or_unknown_fact': 'settled effects with observed outcomes', + 'next_action': 'reconcile the original effects', 'user_input_required': None}, + {'code': 'gate_owner_conflict', 'satisfied': released, + 'missing_or_unknown_fact': 'released work claim or allowed prelaunch custody', + 'next_action': 'inspect and reconcile the current work owner', + 'user_input_required': None}, + ], lifecycle=lifecycle_from_row(row), owner_proof=( + True if (checkpoint_matches and cleanup_matches and candidate_matches + and attempts_settled and effects_settled and released) else None), + legacy_message='gate retry requires a closed, finalized failed gate and released claim') frozen = json.loads(row['candidate_json']) publication = json.loads(row['pr_json'] or 'null') remote = _git(broker.source, 'ls-remote', 'origin', 'refs/heads/' + spec['branch']) @@ -822,14 +861,32 @@ def snapshot(store, run_id, kind=KIND, *, _inventory_profile=None, _observation= and remote and remote.split()[0] == candidate['head'] and not _git(broker.checkout, 'status', '--porcelain', '--untracked-files=no') ) - if (any(candidate.get(k) != frozen.get(k) for k in candidate) - or frozen != state.get('candidate') or not publication_matches - or _git(broker.checkout, 'branch', '--show-current') - != (spec.get('local_branch', spec['branch']) if unpublished else spec['branch']) - or _git(broker.checkout, 'remote', 'get-url', '--push', 'origin') != spec['origin_url'] - or _git(broker.source, 'remote', 'get-url', 'origin') != spec['origin_url'] - or outside_scope(spec['policy'], broker._changed_paths(), checkout=broker.checkout)): - raise ValueError('gate retry lost unchanged owned source or exact published PR head') + candidate_unchanged = (not any(candidate.get(k) != frozen.get(k) for k in candidate) + and frozen == state.get('candidate')) + repository_unchanged = ( + _git(broker.checkout, 'branch', '--show-current') + == (spec.get('local_branch', spec['branch']) if unpublished else spec['branch']) + and _git(broker.checkout, 'remote', 'get-url', '--push', 'origin') == spec['origin_url'] + and _git(broker.source, 'remote', 'get-url', 'origin') == spec['origin_url'] + and not outside_scope(spec['policy'], broker._changed_paths(), checkout=broker.checkout)) + require_observed_predicates(kind, 'delivery_gate_retry', { + 'row': row, 'closed': closed, 'candidate': candidate, 'publication': publication, + }, [ + {'code': 'gate_candidate_changed', 'satisfied': candidate_unchanged, + 'missing_or_unknown_fact': 'unchanged candidate matching the closed checkpoint', + 'next_action': 'inspect the retained candidate and obtain fresh owner proof', + 'user_input_required': None}, + {'code': 'gate_publication_unconfirmed', 'satisfied': bool(publication_matches), + 'missing_or_unknown_fact': 'exact unpublished or published branch and PR identity', + 'next_action': 'reconcile the original publication and branch', + 'user_input_required': None}, + {'code': 'gate_repository_authority_changed', 'satisfied': repository_unchanged, + 'missing_or_unknown_fact': 'original branch, destination and authorized source scope', + 'next_action': 'inspect the retained repository authority and changed paths', + 'user_input_required': None}, + ], lifecycle=lifecycle_from_row(row), owner_proof=( + True if candidate_unchanged and publication_matches and repository_unchanged else None), + legacy_message='gate retry lost unchanged owned source or exact published PR head') load_execution_config(spec) with store._connect() as db: binding = work_binding(store, spec, db) @@ -932,12 +989,18 @@ def admit(store, run_id, payload, *, preflight=False): raise ValueError('gate retry requires an exact zero-repair request') command_digest = digest({'run_id': run_id, **payload}) with store._connect() as db: + db.execute('BEGIN') prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() - if prior: - if prior['request_digest'] != command_digest: - raise ValueError('command ID already belongs to different inputs') - return json.loads(prior['response_json']) + if prior: + from .delivery_lifecycle import command_receipt + + assessment = command_receipt(kind, prior, run_id, command_digest, + request=payload, db=db) + if preflight: + return {**json.loads(prior['response_json']), 'preflight': True, + 'lifecycle_assessment': assessment} + return json.loads(prior['response_json']) seal = snapshot(store, run_id, kind, _observation=preflight) previous = _retained_recovery(store, run_id, seal) if 'verification_test_paths' in payload: @@ -983,12 +1046,14 @@ def admit(store, run_id, payload, *, preflight=False): generation, seal['stage'], preflight_generation=preflight_generation) with _lock(root / 'controller.lock'): with store._connect() as db: + db.execute('BEGIN') prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() - if prior: - if prior['request_digest'] != command_digest: - raise ValueError('command ID already belongs to different inputs') - return json.loads(prior['response_json']) + if prior: + from .delivery_lifecycle import command_receipt + + command_receipt(kind, prior, run_id, command_digest, request=payload, db=db) + return json.loads(prior['response_json']) prepared = None if preflight_generation == 1: from .delivery_gate_preflight_continuation import prepared_handoff diff --git a/runtime/src/devflow_temporal/delivery_lifecycle.py b/runtime/src/devflow_temporal/delivery_lifecycle.py index a9278a5b..014c7f36 100644 --- a/runtime/src/devflow_temporal/delivery_lifecycle.py +++ b/runtime/src/devflow_temporal/delivery_lifecycle.py @@ -5,6 +5,8 @@ """ from __future__ import annotations +import json +import re import sqlite3 from collections.abc import Mapping from functools import partial @@ -12,6 +14,45 @@ VERSION = 1 +# These are observations, not ownership or execution grants. Pause is reserved +# for the pause owner; an unknown state never establishes safe custody. +LIFECYCLE_STATES = frozenset({ + 'pending', 'running', 'stopping', 'stopped', 'recovering', 'terminal', 'unknown', +}) +PAUSE_EXTENSION_STATES = frozenset({'pausing', 'paused', 'resuming'}) + +# Route identities are explicit. Each owner authenticates the full source +# checkpoint and rechecks it when acquiring admission custody. The table does +# not infer a route from an error or authorize a transition by itself. +TRANSITION_TABLE = { + kind: {'from': ('stopped',), 'to': 'recovering'} for kind in ( + 'repair_continuation', 'stopped_delivery_resume', 'published_gate_retry', + 'prepublication_gate_retry', 'published_check_prelaunch_retry', + 'published_ci_retry', 'published_controller_retry', 'recover_publication', + 'retry_prelaunch', 'stopped_resource_closure', 'abandon_pending_resource_closure', + 'published_head_validation', 'investigation_gates_only', 'amend_scope', + ) +} +# Cancelled feature workers and revision-owned completed workers can be resumed +# only when their owner supplies the additional historical authority. +TRANSITION_TABLE['stopped_delivery_resume']['from'] = ('stopped', 'terminal') + + +def lifecycle_from_row(row: Mapping[str, Any]) -> str: + """Describe a persisted projection without treating it as a custody proof.""" + phase, execution, outcome = (row.get(key) for key in ('phase', 'execution_state', 'outcome')) + if execution == 'blocked' and phase == outcome == 'blocked': + return 'stopped' + if execution == 'terminal' and outcome in {'delivered', 'cancelled', 'failed'}: + return 'terminal' + if execution in {'stopping', 'cancelling'}: + return 'stopping' + if execution in {'queued', 'pending', 'running'}: + if row.get('recovery_json'): + return 'recovering' + return 'running' if execution == 'running' else 'pending' + return 'unknown' + # Public command names only; this table never selects a recovery route or grants it. _NEXT_ACTIONS = { 'repair_continuation': 'continue-repair', @@ -61,7 +102,7 @@ def evaluate_transition(facts: Mapping[str, Any], request: Mapping[str, Any]) -> def block(code, fact, action, authorization=False, satisfied=None): blockers.append({'code': code, 'owner': owner, 'missing_or_unknown_fact': fact, 'next_action': action, 'new_authorization_required': authorization, - 'satisfied': satisfied}) + 'satisfied': satisfied, 'user_input_required': None}) if facts.get('owner_proof') is not True: block('owner_proof_unknown', 'authenticated owner and custody proof', @@ -97,11 +138,15 @@ def block(code, fact, action, authorization=False, satisfied=None): 'next_action': predicate['next_action'], 'new_authorization_required': predicate.get('new_authorization_required', False), 'satisfied': predicate.get('satisfied'), + 'user_input_required': predicate.get('user_input_required'), }) + transition = TRANSITION_TABLE.get(operation) observed = {key: facts.get(key) for key in ( 'revision', 'iteration', 'candidate_id', 'candidate_head', 'publication_number')} return {'version': VERSION, 'operation': operation, 'normalized_lifecycle': facts.get('normalized_lifecycle', 'unknown'), + 'transition': ({'from': list(transition['from']), 'to': transition['to']} + if transition is not None else None), 'eligible': not blockers, 'observed_revision': observed['revision'], 'candidate': {'id': observed['candidate_id'], 'head': observed['candidate_head']}, 'publication': ({'number': observed['publication_number']} @@ -121,6 +166,164 @@ def require_transition(facts: Mapping[str, Any], request: Mapping[str, Any]) -> return assessment +def require_observed_predicates(operation: str, owner: str, proof: Any, + predicates: list[dict[str, Any]], *, + lifecycle: str = 'unknown', owner_proof: bool | None = True, + legacy_message: str | None = None + ) -> dict[str, Any]: + """Refuse exact observed predicates in both preflight and owner admission. + + This helper never substitutes a digest for custody: callers must establish + the authenticated observation and all owner-specific predicates themselves. + """ + from .contracts import digest + + try: + return require_transition({ + 'operation': operation, 'owner': owner, 'owner_proof': owner_proof, + 'normalized_lifecycle': lifecycle, 'proof_digest': digest(proof), + 'comparisons': [], 'predicates': predicates, + }, {}) + except RecoveryRefused as exc: + if legacy_message is not None: + exc.args = (legacy_message,) + raise + + +def _event_backed_receipt(db, operation, run_id, request, response): + """Bind a gate/publication command to its retained admission and event.""" + from .contracts import canonical_json + + pending = operation == 'recover_publication' + expected_kind = 'pending_publication_retry' if pending else operation + saved = db.execute('SELECT recovery_json FROM delivery_runs WHERE run_id=?', + (run_id,)).fetchone() + roots = [saved[0]] if saved and saved[0] else [] + if not pending: + gate = db.execute('SELECT recovery_json FROM delivery_gate_admissions WHERE run_id=?', + (run_id,)).fetchone() + if gate and gate[0]: + roots.append(gate[0]) + revisions = set() + try: + for raw in roots: + recovery = json.loads(raw) + while isinstance(recovery, dict): + if (recovery.get('kind') == expected_kind + and canonical_json(recovery.get('command')) == canonical_json(request)): + revision = recovery.get('seal', {}).get('row', {}).get('revision') + if type(revision) is int: + revisions.add(revision + 1) + recovery = recovery.get('original_recovery') + if len(revisions) != 1: + return False + event = db.execute( + 'SELECT payload_json FROM delivery_events WHERE run_id=? AND type=? AND ' + 'run_revision=?', + (run_id, 'publication_recovery_queued' if pending else 'gates_retry_queued', + revisions.pop()), + ).fetchall() + return (len(event) == 1 + and canonical_json(json.loads(event[0][0])) == canonical_json(response)) + except (TypeError, ValueError, AttributeError): + return False + + +def _receipt_matches_owner(db, operation, run_id, request_digest, request, response): + """Check historical route evidence without comparing to the current workflow.""" + if not isinstance(response, dict) or response.get('run_id') != run_id: + return False + workflow = response.get('workflow_id') + if not isinstance(workflow, str) or not workflow: + return False + if operation in {'stopped_delivery_resume', 'repair_continuation'}: + if (response.get('phase') != 'repair_continuation_queued' + or not isinstance(response.get('dashboard_url'), str) + or not response['dashboard_url'].endswith('/runs/' + run_id) + or type(response.get('authorized_through_iteration')) is not int): + return False + if operation == 'repair_continuation': + maximum = request['expected_iteration'] + request['additional_iterations'] + grant = db.execute( + 'SELECT granted_iterations,maximum_iteration FROM delivery_repair_grants ' + 'WHERE run_id=? AND command_id=?', (run_id, request['command_id']), + ).fetchone() + return (workflow == f'delivery-{run_id}-repair-continuation-1' + and response.get('existing') is False and grant is not None + and grant['granted_iterations'] == request['additional_iterations'] + and grant['maximum_iteration'] == maximum + and response['authorized_through_iteration'] == maximum) + if (workflow != f'delivery-{run_id}-resume-{request_digest[:20]}' + or not isinstance(response.get('admission_sha256'), str) + or not re.fullmatch(r'[0-9a-f]{64}', response['admission_sha256'])): + return False + try: + events = [json.loads(item[0]) for item in db.execute( + "SELECT payload_json FROM delivery_events WHERE run_id=? " + "AND type='stopped_resume_queued'", (run_id,))] + except (TypeError, ValueError): + return False + admitted = [item for item in events if isinstance(item, dict) + and item.get('command_id') == request['command_id']] + return (len(admitted) == 1 + and type(admitted[0].get('authorized_through_iteration')) is int + and admitted[0]['authorized_through_iteration'] + == response['authorized_through_iteration']) + if operation == 'recover_publication': + valid = (response.get('phase') == 'publication_recovery_queued' + and response.get('implementation_authority') is False) + elif operation in {'published_gate_retry', 'prepublication_gate_retry', + 'published_check_prelaunch_retry', 'published_ci_retry', + 'published_controller_retry'}: + valid = (response.get('phase') == 'gates_retry_queued' + and response.get('implementation_authority') is False + and type(response.get('additional_iterations')) is int + and response['additional_iterations'] == 0 + and response.get('candidate_id') == request['expected_candidate_id']) + else: + return False + return valid and _event_backed_receipt(db, operation, run_id, request, response) + + +def command_receipt(operation: str, prior: Mapping[str, Any], run_id: str, + request_digest: str, *, request: Mapping[str, Any], db) -> dict[str, Any]: + """Describe a prior admission only when its route-specific receipt is intact.""" + from .contracts import digest + + same = prior['request_digest'] == request_digest + try: + response = json.loads(prior['response_json']) + except (TypeError, ValueError): + response = None + intact = (same and prior['run_id'] == run_id + and _receipt_matches_owner(db, operation, run_id, request_digest, request, response)) + code = ('already_admitted' if intact else 'command_conflict' if not same + else 'command_receipt_unknown') + fact = ('an existing admission for these exact inputs' if intact else + 'a command ID bound to these exact inputs' if not same else + 'an intact saved admission receipt for this run') + assessment = evaluate_transition({ + 'operation': operation, 'owner': 'delivery_commands', + 'owner_proof': True if intact or not same else None, + 'proof_digest': digest({'run_id': prior['run_id'], 'prior': prior['request_digest'], + 'response': prior['response_json'], 'request': request_digest}), + 'comparisons': [], 'predicates': [{ + 'code': code, 'satisfied': False if intact or not same else None, + 'missing_or_unknown_fact': fact, + 'next_action': ('inspect the existing admission' if intact else + 'inspect the saved command and obtain a fresh assessment'), + 'user_input_required': False if intact else None, + }], + }, {}) + assessment['disposition'] = code + if not intact: + exc = RecoveryRefused(assessment) + exc.args = ('command ID already belongs to different inputs' if not same else + 'stored command receipt is unavailable',) + raise exc + return assessment + + def require_same_proof(operation: str, owner: str, observed_digest: str, expected_digest: str) -> dict[str, Any]: """Recheck the owner's full seal under its admission custody.""" diff --git a/runtime/src/devflow_temporal/delivery_pending_publication.py b/runtime/src/devflow_temporal/delivery_pending_publication.py index 1c44625e..8cbd3799 100644 --- a/runtime/src/devflow_temporal/delivery_pending_publication.py +++ b/runtime/src/devflow_temporal/delivery_pending_publication.py @@ -29,7 +29,7 @@ def _chain(recovery): recovery = recovery.get('original_recovery') -def _refuse(code, fact, action, proof, *, message=None): +def _refuse(code, fact, action, proof, *, message=None, satisfied=False): from .delivery_lifecycle import RecoveryRefused, require_transition try: @@ -37,7 +37,7 @@ def _refuse(code, fact, action, proof, *, message=None): 'operation': 'recover_publication', 'owner': 'pending_publication', 'owner_proof': True, 'normalized_lifecycle': 'stopped', 'proof_digest': digest(proof), 'comparisons': [], 'predicates': [{ - 'code': code, 'satisfied': False, 'missing_or_unknown_fact': fact, + 'code': code, 'satisfied': satisfied, 'missing_or_unknown_fact': fact, 'next_action': action, }], }, {}) @@ -231,8 +231,18 @@ def snapshot(store, run_id, *, observation=False): message='original publication executor or remote outcome remains unknown') from .delivery_resources import observe_completed_resources - settlement = {'publication': publication, - 'native_cleanup': observe_completed_resources(spec, attempts)} + try: + native_cleanup = observe_completed_resources(spec, attempts) + except (ValueError, OSError) as exc: + from .delivery_lifecycle import RecoveryRefused + + if isinstance(exc, RecoveryRefused): + raise + _refuse('publication_native_custody_unknown', + 'settled native actors, roots and resource leases', + 'reconcile the original native resource custody', + {'row': row, 'closed': closed}, message=str(exc), satisfied=None) + settlement = {'publication': publication, 'native_cleanup': native_cleanup} with store._connect() as db: binding = work_binding(store, spec, db) return {'row': _sealed_row(row), 'closed': closed, 'spec': spec, @@ -257,17 +267,22 @@ def admit(store, run_id, payload, *, preflight=False): r'[0-9a-f]{' + str(n) + '}', payload[k]) for k, n in (('expected_candidate_id', 64), ('expected_head', 40)))): raise ValueError('invalid pending publication retry identity') + from .delivery_lifecycle import command_receipt, require_same_proof, require_transition + command_digest = digest({'run_id': run_id, **payload}) with store._connect() as db: + db.execute('BEGIN') prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() - if prior: - if prior['request_digest'] != command_digest: - raise ValueError('command ID already belongs to different inputs') - return json.loads(prior['response_json']) + if prior: + assessment = command_receipt('recover_publication', prior, run_id, command_digest, + request=payload, db=db) + if preflight: + return {**json.loads(prior['response_json']), 'preflight': True, + 'lifecycle_assessment': assessment} + return json.loads(prior['response_json']) seal = snapshot(store, run_id, observation=preflight) previous = _retained_recovery(store, run_id, seal) - from .delivery_lifecycle import require_same_proof, require_transition assessment = require_transition({ 'operation': 'recover_publication', 'owner': 'pending_publication', 'owner_proof': True, @@ -287,12 +302,13 @@ def admit(store, run_id, payload, *, preflight=False): root = Path(spec['state_dir']) / namespace with _lock(root / 'controller.lock'): with store._connect() as db: + db.execute('BEGIN') prior = db.execute('SELECT * FROM delivery_commands WHERE command_id=?', (payload['command_id'],)).fetchone() - if prior: - if prior['request_digest'] != command_digest: - raise ValueError('command ID already belongs to different inputs') - return json.loads(prior['response_json']) + if prior: + command_receipt('recover_publication', prior, run_id, command_digest, + request=payload, db=db) + return json.loads(prior['response_json']) execution = prepare_runtime(spec, root, command_digest, digest(seal)) require_same_proof('recover_publication', 'pending_publication', digest(snapshot(store, run_id)), digest(seal)) diff --git a/runtime/src/devflow_temporal/delivery_stopped_resume.py b/runtime/src/devflow_temporal/delivery_stopped_resume.py index 2b64c72c..208480b6 100644 --- a/runtime/src/devflow_temporal/delivery_stopped_resume.py +++ b/runtime/src/devflow_temporal/delivery_stopped_resume.py @@ -201,7 +201,14 @@ def worker_allowance(spec, iteration, iterations, *, credit=None): owner = shared.token(current) shared.repair_allowance(owner, reference, current=True) if iterations > budget['maximum'] - budget['used'] + int(bool(credit)): - raise ValueError('feature product repair limit exhausted') + from .delivery_lifecycle import require_observed_predicates + + require_observed_predicates(KIND, 'feature_registry', {'owner': owner, 'budget': budget}, [{ + 'code': 'product_budget_exhausted', 'satisfied': False, + 'missing_or_unknown_fact': 'remaining authenticated feature product repair allowance', + 'next_action': 'inspect the original feature repair allowance', + 'new_authorization_required': True, 'user_input_required': None, + }], lifecycle='unknown', legacy_message='feature product repair limit exhausted') return {'run_id': spec['run_id'], 'owner': owner, 'receipt': reference, 'start_iteration': iteration, 'maximum_iteration': iteration + iterations, **({'credit': credit} if credit else {})} @@ -1001,19 +1008,38 @@ def _implementation_receipt(spec, role, attempt): def _candidate(broker, state, attempts, *, missing_rollout=None, input_rejection=None, - recovery=None, pending=None, quarantined=None, historical_quarantines=()): + recovery=None, pending=None, quarantined=None, historical_quarantines=(), + typed_refusal=False, lifecycle='unknown'): candidate = broker.candidate() + + def refuse(code, fact, action, message, *, satisfied=None): + if typed_refusal: + from .delivery_lifecycle import require_observed_predicates + + require_observed_predicates(KIND, 'delivery_stopped_resume', { + 'closed_candidate': state.get('candidate'), 'current_candidate': candidate, + }, [{ + 'code': code, 'satisfied': satisfied, 'missing_or_unknown_fact': fact, + 'next_action': action, 'user_input_required': None, + }], lifecycle=lifecycle, owner_proof=None, legacy_message=message) + raise ValueError(message) + implementations = [r for r in state.get('roles', []) if r.get('role') == 'implement'] latest = implementations[-1] if implementations else None retained_unknown = (latest is not None and pending and pending.get('kind') == 'unstarted_preflight' and digest(latest) in {p['closed_role_sha256'] for p in historical_quarantines}) if latest and latest.get('status') != 'pass': - if ((latest.get('cleanup') != 'confirmed' and not (quarantined or retained_unknown)) - or (latest.get('candidate') != candidate - and not _historical_candidate( - broker, state, candidate, latest, recovery, pending))): - raise ValueError('blocked implementation source or session is unconfirmed') + if latest.get('cleanup') != 'confirmed' and not (quarantined or retained_unknown): + refuse('stopped_implementation_cleanup_unknown', + 'confirmed blocked implementation cleanup or its exact quarantine', + 'reconcile the original implementation custody', + 'blocked implementation source or session is unconfirmed') + if (latest.get('candidate') != candidate + and not _historical_candidate(broker, state, candidate, latest, recovery, pending)): + refuse('stopped_candidate_changed', 'unchanged or authenticated historical candidate', + 'inspect the retained candidate and obtain fresh owner proof', + 'blocked implementation source or session is unconfirmed') matches = [a for a in attempts if a['role'] == 'implement' and a['iteration'] == latest['iteration'] and a['session_id'] == latest['session_id'] @@ -1027,35 +1053,54 @@ def _candidate(broker, state, attempts, *, missing_rollout=None, input_rejection # A closed Temporal activity authenticates controller-only preparation failures. # Every provider or supervisor prelaunch receipt must also match its durable attempt. if len(matches) != 1 and not no_provider and not (quarantined or retained_unknown): - raise ValueError('blocked implementation lacks its exact supervisor receipt') + refuse('stopped_implementation_receipt_unproven', + 'one exact supervisor receipt for the blocked implementation', + 'inspect the original implementation attempt and receipt', + 'blocked implementation lacks its exact supervisor receipt') if latest.get('session_id') is None and not no_provider and ( latest.get('finish_reason') != 'prelaunch' and not (missing_rollout or input_rejection or quarantined or retained_unknown)): - raise ValueError('blocked implementation has no authenticated launch outcome') + refuse('stopped_launch_outcome_unproven', + 'authenticated prelaunch or provider outcome for the blocked implementation', + 'inspect the original launch and provider custody', + 'blocked implementation has no authenticated launch outcome') elif candidate != state.get('candidate'): - raise ValueError('stopped candidate changed after its sealed checkpoint') - if (any((r.get('cleanup') != 'confirmed' or r.get('finish_reason') == 'recovery_unknown') - and not (quarantined and r is latest) - and digest(r) not in {p['closed_role_sha256'] for p in historical_quarantines} - for r in state.get('roles', [])) - or any(a['state'] != 'finished' or a['cleanup'] != 'confirmed' for a in attempts)): - raise ValueError('stopped role ownership is unknown') + refuse('stopped_candidate_changed', 'unchanged candidate at the closed checkpoint', + 'inspect the retained candidate and obtain fresh owner proof', + 'stopped candidate changed after its sealed checkpoint', satisfied=False) + if any((r.get('cleanup') != 'confirmed' or r.get('finish_reason') == 'recovery_unknown') + and not (quarantined and r is latest) + and digest(r) not in {p['closed_role_sha256'] for p in historical_quarantines} + for r in state.get('roles', [])): + refuse('stopped_role_cleanup_unknown', 'confirmed role cleanup or exact quarantine', + 'reconcile the original role custody', 'stopped role ownership is unknown') + if any(a['state'] != 'finished' or a['cleanup'] != 'confirmed' for a in attempts): + refuse('stopped_attempt_custody_unknown', 'finished attempts and confirmed cleanup', + 'reconcile the original attempt custody', 'stopped role ownership is unknown') sessions = {r.get('session_id') for r in implementations if r.get('session_id')} if len(sessions) > 1: - raise ValueError('original implementation session changed') + refuse('stopped_session_conflict', 'one original implementation session', + 'inspect the retained implementation session history', + 'original implementation session changed', satisfied=False) if not sessions and any(a['role'] == 'implement' and a['session_id'] for a in attempts): - raise ValueError('unrecorded implementation session exists') + refuse('stopped_session_unrecorded', 'matching role and attempt session identity', + 'inspect the retained role and attempt session evidence', + 'unrecorded implementation session exists', satisfied=False) changed = broker._changed_paths(broker.spec['base_sha']) from .delivery_source_scope import outside_scope if outside_scope(broker.spec['policy'], changed, checkout=broker.checkout): - raise ValueError('stopped source escaped its frozen scope') + refuse('stopped_scope_changed', 'source paths within the original authorized scope', + 'inspect changed paths and the original scope authority', + 'stopped source escaped its frozen scope', satisfied=False) _git(broker.checkout, 'merge-base', '--is-ancestor', broker.spec['base_sha'], candidate['head']) if (_git(broker.checkout, 'branch', '--show-current') != broker.spec.get('local_branch', broker.spec['branch']) or _git(broker.checkout, 'remote', 'get-url', '--push', 'origin') != broker.spec['origin_url']): - raise ValueError('stopped checkout branch or origin changed') + refuse('stopped_repository_changed', 'original checkout branch and push destination', + 'inspect the original repository and branch authority', + 'stopped checkout branch or origin changed', satisfied=False) return candidate, next(iter(sessions), None) @@ -1108,25 +1153,75 @@ def _snapshot(store, run_id, *, _revision=None, review_subject=False, _observati raise ValueError('stopped worker cleanup belongs to a different closed execution') cleanup_authority = authenticated state_checks, row_checks = closed_checks(row, state, cleanup_authority) - if (row['phase'] not in stopped or row['outcome'] != row['phase'] - or row['execution_state'] != expected_execution or row['cleanup'] != 'confirmed' - or state.get('run_id') != run_id or state.get('phase') != row['phase'] - or state.get('outcome') != row['outcome'] - or (state.get('cleanup') != 'confirmed' and cleanup_authority is None) - or state.get('execution_state') != expected_execution - or closed.get('workflow_id') != predecessor_id - or state.get('candidate') != json.loads(row['candidate_json'] or 'null') - or state_checks != row_checks - or state.get('pull_request') != json.loads(row['pr_json'] or 'null') - or state.get('revision') != row['protocol_revision'] - or state.get('iteration') != row['iteration'] or state.get('error') != row['error'] - or closed['request_digest'] != row['request_digest'] - or closed['recovery_digest'] != (digest(previous) if previous else None) - or claim is not None - or any(e['state'] != 'complete' or e['observed_json'] is None for e in effects)): - raise ValueError('resume requires a closed finalized delivery with released ownership') - if not isinstance(spec.get('accepted_plan'), str) or not spec['accepted_plan'].strip(): - raise ValueError('implementation resume requires an accepted plan') + # Keep the closed verdict immutable while reporting later custody facts + # separately. These are exactly the facts checked by the original guard. + from .delivery_lifecycle import ( + RecoveryRefused, + lifecycle_from_row, + require_observed_predicates, + ) + + status_matches = (row['phase'] in stopped and row['outcome'] == row['phase'] + and row['execution_state'] == expected_execution and state.get('run_id') == run_id + and state.get('phase') == row['phase'] and state.get('outcome') == row['outcome'] + and state.get('execution_state') == expected_execution) + cleanup_matches = (row['cleanup'] == 'confirmed' + and (state.get('cleanup') == 'confirmed' or cleanup_authority is not None)) + identity_matches = (closed.get('workflow_id') == predecessor_id + and closed['request_digest'] == row['request_digest'] + and closed['recovery_digest'] == (digest(previous) if previous else None)) + checkpoint_matches = (state.get('candidate') == json.loads(row['candidate_json'] or 'null') + and state_checks == row_checks + and state.get('pull_request') == json.loads(row['pr_json'] or 'null') + and state.get('revision') == row['protocol_revision'] + and state.get('iteration') == row['iteration'] and state.get('error') == row['error']) + plan_available = (isinstance(spec.get('accepted_plan'), str) + and bool(spec['accepted_plan'].strip())) + effects_settled = all(e['state'] == 'complete' and e['observed_json'] is not None + for e in effects) + try: + require_observed_predicates( + KIND, 'delivery_stopped_resume', {'row': row, 'closed': closed}, [ + {'code': 'stopped_state_unproven', 'satisfied': status_matches, + 'missing_or_unknown_fact': 'matching stopped or supported terminal run state', + 'next_action': 'inspect the retained run and closed predecessor', + 'user_input_required': None}, + {'code': 'stopped_cleanup_unconfirmed', 'satisfied': True if cleanup_matches else None, + 'missing_or_unknown_fact': 'confirmed cleanup or authenticated later reconciliation', + 'next_action': 'reconcile the original cleanup and custody', + 'user_input_required': None}, + {'code': 'stopped_predecessor_unproven', 'satisfied': identity_matches, + 'missing_or_unknown_fact': 'matching workflow, admission and recovery identities', + 'next_action': 'inspect the closed predecessor and its original admission', + 'user_input_required': None}, + {'code': 'stopped_checkpoint_changed', 'satisfied': checkpoint_matches, + 'missing_or_unknown_fact': 'unchanged candidate, checks, publication and verdict', + 'next_action': 'inspect retained checkpoint and immutable history', + 'user_input_required': None}, + {'code': 'stopped_owner_conflict', 'satisfied': claim is None, + 'missing_or_unknown_fact': 'released original work claim', + 'next_action': 'inspect and reconcile the current work owner', + 'user_input_required': None}, + {'code': 'stopped_effects_unsettled', 'satisfied': True if effects_settled else None, + 'missing_or_unknown_fact': 'settled effects with observed outcomes', + 'next_action': 'reconcile the original effects', 'user_input_required': None}, + {'code': 'accepted_plan_unavailable', 'satisfied': plan_available, + 'missing_or_unknown_fact': 'original accepted plan for implementation resume', + 'next_action': 'inspect retained intake and preparation; no implementation resume ' + 'is authorized without an accepted plan', + 'user_input_required': None}, + ], lifecycle=lifecycle_from_row(row), owner_proof=( + True if (status_matches and cleanup_matches and identity_matches and checkpoint_matches + and claim is None and effects_settled) else None), + legacy_message=('resume requires a closed finalized delivery with released ownership' + if not (status_matches and cleanup_matches and identity_matches + and checkpoint_matches and claim is None and effects_settled) + else 'implementation resume requires an accepted plan')) + except RecoveryRefused as exc: + if review_subject: + # Other owners must report their own operation, not a resume refusal. + raise ValueError(str(exc)) from exc + raise if spec.get('baseline_checks_version') in (1, 2): baseline = state.get('checks', {}).get('baseline', {}) required = {check['id'] for check in spec['policy']['baseline_checks']} @@ -1182,7 +1277,8 @@ def _snapshot(store, run_id, *, _revision=None, review_subject=False, _observati candidate, session = _candidate( broker, state, attempts, missing_rollout=(session_custody or {}).get('missing_rollout'), input_rejection=(session_custody or {}).get('input_rejection'), - recovery=previous, pending=pending, quarantined=quarantined, historical_quarantines=history) + recovery=previous, pending=pending, quarantined=quarantined, historical_quarantines=history, + typed_refusal=not review_subject, lifecycle=lifecycle_from_row(row)) publication = state.get('pull_request') if publication: published_identity(broker, candidate, publication) @@ -1243,12 +1339,18 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): raise ValueError('stopped resume requires a finite exact-checkpoint command') command_digest = digest({'run_id': run_id, **command}) with store._connect() as db: - prior = db.execute('SELECT request_digest,response_json FROM delivery_commands ' + db.execute('BEGIN') + prior = db.execute('SELECT run_id,request_digest,response_json FROM delivery_commands ' 'WHERE command_id=?', (command['command_id'],)).fetchone() - if prior: - if prior[0] != command_digest: - raise ValueError('command ID already belongs to different inputs') - return json.loads(prior[1]) + if prior: + from .delivery_lifecycle import command_receipt + + assessment = command_receipt(KIND, prior, run_id, command_digest, + request=command, db=db) + if preflight: + return {**json.loads(prior['response_json']), 'preflight': True, + 'lifecycle_assessment': assessment} + return json.loads(prior['response_json']) seal = snapshot(store, run_id, _revision=_revision, _observation=preflight) gate_only = None if _revision is not None: @@ -1258,11 +1360,11 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): with store._connect() as db: pending = None if gate_only else pending_repair(seal, db=db) ci_findings = None if gate_only else sealed_ci_findings(seal) - from .delivery_lifecycle import require_same_proof, require_transition + from .delivery_lifecycle import lifecycle_from_row, require_same_proof, require_transition lifecycle_assessment = require_transition({ 'operation': KIND, 'owner': 'delivery_stopped_resume', 'owner_proof': True, - 'normalized_lifecycle': 'stopped', 'proof_digest': digest(seal), + 'normalized_lifecycle': lifecycle_from_row(seal['row']), 'proof_digest': digest(seal), 'revision': seal['state']['revision'], 'iteration': seal['state']['iteration'], 'candidate_id': seal['candidate']['id'], 'candidate_head': seal['candidate']['head'], @@ -1271,15 +1373,28 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): interrupted = None if _revision is not None or pending else interrupted_resume(seal) if interrupted and interrupted['additional_iterations'] != command['additional_iterations']: interrupted = None - if command['additional_iterations'] == 0 and not (pending or gate_only or interrupted): - raise ValueError('zero additional iterations require an authenticated pending repair ' - 'or an interrupted iteration') - if seal.get('quarantined_implementation') and command['additional_iterations'] == 0: - raise ValueError('unknown implementation requires a separately authorized new iteration') maximum = continuation_maximum(seal['state']['iteration'], command['additional_iterations'], pending) - if maximum is None: - raise ValueError('unstarted preflight retains only its existing finite ceiling') + from .delivery_lifecycle import require_observed_predicates + + require_observed_predicates(KIND, 'delivery_stopped_resume', seal, [ + {'code': 'pending_repair_unproven', + 'satisfied': (command['additional_iterations'] != 0 + or bool(pending or gate_only or interrupted)), + 'missing_or_unknown_fact': 'authenticated pending repair or interrupted iteration', + 'next_action': 'inspect the retained repair admission and original iteration', + 'user_input_required': None}, + {'code': 'unknown_implementation_requires_new_turn', + 'satisfied': (not seal.get('quarantined_implementation') + or command['additional_iterations'] != 0), + 'missing_or_unknown_fact': 'separately authorized new implementation iteration', + 'next_action': 'review the unknown implementation and its original allowance', + 'new_authorization_required': True, 'user_input_required': True}, + {'code': 'pending_repair_ceiling_changed', 'satisfied': maximum is not None, + 'missing_or_unknown_fact': 'unchanged finite pending preflight ceiling', + 'next_action': 'inspect the retained preflight admission and its ceiling', + 'user_input_required': None}, + ], lifecycle=lifecycle_from_row(seal['row'])) credit = None if _revision is not None: from .delivery_feature_execution import registry @@ -1299,7 +1414,13 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): and not fixed_budget_allows(budget_spec, seal['state']['iteration'], command['additional_iterations'], pending=bool(pending or gate_only or interrupted))): - raise ValueError('a fixed repair budget cannot receive additional iterations') + require_observed_predicates(KIND, 'delivery_stopped_resume', seal, [{ + 'code': 'product_budget_exhausted', 'satisfied': False, + 'missing_or_unknown_fact': 'remaining original finite product repair allowance', + 'next_action': 'inspect the original repair budget and retained admission', + 'new_authorization_required': True, 'user_input_required': None, + }], lifecycle=lifecycle_from_row(seal['row']), + legacy_message='a fixed repair budget cannot receive additional iterations') amended = scope_spec(store, seal, command) browser_amendment = None if BROWSER_FIELDS <= command.keys(): @@ -1349,13 +1470,16 @@ def admit(store, run_id, command, *, preflight=False, _revision=None): root = Path(spec['state_dir']) / 'stopped-resumes' / command_digest with _lock(Path(spec['state_dir']) / 'stopped-resumes' / 'controller.lock'): with store._connect() as db: + db.execute('BEGIN') prior = db.execute( - 'SELECT request_digest,response_json FROM delivery_commands WHERE command_id=?', + 'SELECT run_id,request_digest,response_json FROM delivery_commands ' + 'WHERE command_id=?', (command['command_id'],)).fetchone() - if prior: - if prior[0] != command_digest: - raise ValueError('command ID already belongs to different inputs') - return json.loads(prior[1]) + if prior: + from .delivery_lifecycle import command_receipt + + command_receipt(KIND, prior, run_id, command_digest, request=command, db=db) + return json.loads(prior['response_json']) # Recheck all stopped readbacks after lock acquisition, before any native work. require_same_proof(KIND, 'delivery_stopped_resume', digest(snapshot(store, run_id, _revision=_revision)), digest(seal)) diff --git a/runtime/src/devflow_temporal/delivery_store.py b/runtime/src/devflow_temporal/delivery_store.py index 6bbf5bb0..712bd967 100644 --- a/runtime/src/devflow_temporal/delivery_store.py +++ b/runtime/src/devflow_temporal/delivery_store.py @@ -1855,7 +1855,37 @@ def continue_repair( or not fixed_budget_allows(original_budget, supplied.get('expected_iteration'), supplied['additional_iterations'])): - raise ValueError("a fixed repair budget cannot receive additional iterations") + from .delivery_lifecycle import require_observed_predicates + + requested_kind = supplied.get('continuation_kind') + with self._connect() as db: + observed = db.execute( + 'SELECT iteration,protocol_revision FROM delivery_runs WHERE run_id=?', + (run_id,)).fetchone() + iteration_matches = (observed is not None + and type(supplied.get('expected_iteration')) is int + and observed['iteration'] == supplied['expected_iteration']) + code = ('additional_iterations_not_supported' if requested_kind != KIND else + 'product_budget_exhausted' if iteration_matches else + 'budget_checkpoint_unknown') + if requested_kind != KIND: + fact = 'a recovery route that supports product iterations' + elif iteration_matches: + fact = 'remaining original finite product repair allowance' + else: + fact = 'current iteration bound to the requested budget checkpoint' + require_observed_predicates( + requested_kind or 'repair_continuation', 'delivery_budget', + {'original_authority': original_budget, 'request': supplied, + 'observed_iteration': observed['iteration'] if observed else None}, [{ + 'code': code, + 'satisfied': False if requested_kind != KIND or iteration_matches else None, + 'missing_or_unknown_fact': fact, + 'next_action': 'inspect the original budget and recovery checkpoint', + 'new_authorization_required': code == 'product_budget_exhausted', + 'user_input_required': None, + }], lifecycle='unknown', + legacy_message='a fixed repair budget cannot receive additional iterations') if (isinstance(supplied, dict) and supplied.get('continuation_kind') == 'stopped_delivery_resume'): from .delivery_stopped_resume import admit @@ -1921,6 +1951,11 @@ def continue_repair( return json.loads(prior["response_json"]) raise ValueError("repair continuation fields do not match the contract") from .delivery_broker import DeliveryBroker + from .delivery_lifecycle import ( + command_receipt, + require_observed_predicates, + require_transition, + ) from .delivery_repair import ( confirmed_native_cleanup, failed_gate_diagnostics, @@ -1952,13 +1987,17 @@ def continue_repair( raise ValueError("invalid repair continuation identity or grant") command_digest = digest({"run_id": run_id, **supplied}) with self._connect() as db: + db.execute("BEGIN") prior = db.execute( - "SELECT request_digest,response_json FROM delivery_commands WHERE command_id=?", - (command_id,), + "SELECT run_id,request_digest,response_json FROM delivery_commands " + "WHERE command_id=?", (command_id,), ).fetchone() if prior: - if prior["request_digest"] != command_digest: - raise ValueError("command ID already belongs to different inputs") + receipt = command_receipt('repair_continuation', prior, run_id, command_digest, + request=supplied, db=db) + if preflight: + return {**json.loads(prior["response_json"]), "preflight": True, + "lifecycle_assessment": receipt} return json.loads(prior["response_json"]) row = db.execute("SELECT * FROM delivery_runs WHERE run_id=?", (run_id,)).fetchone() granted = db.execute( @@ -1967,7 +2006,13 @@ def continue_repair( if row is None: raise ValueError("run ID not found") if granted: - raise ValueError("this run already received its one repair grant") + require_observed_predicates('repair_continuation', 'repair_admission', + {'row': dict(row), 'grant_present': True}, [{ + 'code': 'repair_grant_consumed', 'satisfied': False, + 'missing_or_unknown_fact': 'unused original repair grant', + 'next_action': 'inspect the original repair admission', + }], lifecycle='unknown', + legacy_message='this run already received its one repair grant') spec = self.effective_spec(run_id) finalized = (spec.get("resource_cleanup_version") == 1 and row["cleanup"] == "confirmed") @@ -1986,8 +2031,6 @@ def continue_repair( roles = state.get("roles") checks = state.get("checks") iteration = state.get("iteration") - from .delivery_lifecycle import require_transition - facts = { "operation": "repair_continuation", "owner": "repair_admission", "owner_proof": True, "normalized_lifecycle": "stopped", @@ -2055,22 +2098,47 @@ def continue_repair( for role in roles ) ): - raise ValueError("original implementer session or role cleanup is unconfirmed") + require_observed_predicates('repair_continuation', 'repair_admission', + {'row': dict(row), 'closed': closed}, [{ + 'code': 'repair_role_custody_unknown', 'satisfied': None, + 'missing_or_unknown_fact': 'passed implementation session and confirmed ' + 'role cleanup', + 'next_action': 'inspect the original implementation and role custody', + }], lifecycle='stopped', owner_proof=None, + legacy_message='original implementer session or role cleanup is unconfirmed') for key in ("prepublish", "local", "browser_qa"): result = checks.get(key) if result is None: continue if not isinstance(result, dict) or result.get("state") == "unknown": - raise ValueError("a check has an unknown outcome") + require_observed_predicates('repair_continuation', 'repair_admission', + {'row': dict(row), 'closed': closed}, [{ + 'code': 'repair_check_outcome_unknown', 'satisfied': None, + 'missing_or_unknown_fact': f'known {key} check outcome', + 'next_action': 'inspect the original check outcome', + }], lifecycle='stopped', owner_proof=None, + legacy_message='a check has an unknown outcome') if result.get("cleanup") == "unknown" or ( key == "browser_qa" and result.get("cleanup") != "confirmed" ): - raise ValueError("a check has unconfirmed cleanup") + require_observed_predicates('repair_continuation', 'repair_admission', + {'row': dict(row), 'closed': closed}, [{ + 'code': 'repair_check_cleanup_unknown', 'satisfied': None, + 'missing_or_unknown_fact': f'confirmed {key} check cleanup', + 'next_action': 'reconcile the original check cleanup', + }], lifecycle='stopped', owner_proof=None, + legacy_message='a check has unconfirmed cleanup') if any( not isinstance(item, dict) or item.get("cleanup") != "confirmed" for item in result.get("results", []) ): - raise ValueError("a check child has unconfirmed cleanup") + require_observed_predicates('repair_continuation', 'repair_admission', + {'row': dict(row), 'closed': closed}, [{ + 'code': 'repair_check_child_cleanup_unknown', 'satisfied': None, + 'missing_or_unknown_fact': f'confirmed {key} check child cleanup', + 'next_action': 'reconcile the original check child custody', + }], lifecycle='stopped', owner_proof=None, + legacy_message='a check child has unconfirmed cleanup') findings = failed_gate_diagnostics(state, spec) if not findings: raise ValueError("failed gate supplied no repair diagnostics") @@ -2108,15 +2176,95 @@ def continue_repair( if finalized: recovery.update(original_recovery=previous_recovery, predecessor_spec=spec, finalized_checkpoint=True, cleanup_digest=cleanup_digest) - if preflight: - from .delivery_policy_recovery import work_binding + from .delivery_policy_recovery import work_binding - with self._connect() as db: + def observe_admission(db): + """Check the same frozen inventory before preview and under admission custody.""" + current = db.execute("SELECT * FROM delivery_runs WHERE run_id=?", (run_id,)).fetchone() + claim = self.state.claim_for(db, spec["work_id"]) + attempts = db.execute( + "SELECT role,iteration,state,session_id,cleanup " + "FROM delivery_attempts WHERE run_id=?", (run_id,), + ).fetchall() + effects = db.execute( + "SELECT kind,state,observed_json FROM delivery_effects WHERE run_id=?", (run_id,), + ).fetchall() + grant = db.execute( + "SELECT 1 FROM delivery_repair_grants WHERE run_id=?", (run_id,), + ).fetchone() + checkpoint = (current is not None + and current["request_json"] == row["request_json"] + and current["workflow_id"] == row["workflow_id"] + and current["recovery_json"] == row["recovery_json"] + and current["phase"] == "blocked" and current["outcome"] == "blocked" + and current["execution_state"] == "blocked" + and current["cleanup"] in ({"none", "confirmed"} if stopped_claim else {"none"}) + and current["error"] == state["error"] + and current["protocol_revision"] == state["revision"] + and current["iteration"] == iteration + and json.loads(current["candidate_json"] or "null") == candidate + and json.loads(current["pr_json"] or "null") == pr + and json.loads(current["checks_json"] or "{}") == checks) + owner = (claim is None if stopped_claim else + claim is not None and claim["owner"] == f"external:devflow:{run_id}") + attempt_custody = (len(attempts) == len(roles) + and all(item["state"] == "finished" and item["cleanup"] == "confirmed" + for item in attempts) + and sorted((item["role"], item["iteration"], item["session_id"]) + for item in attempts) + == sorted((item["role"], item["iteration"], item["session_id"]) + for item in roles)) + effect_custody = (any(item["kind"] == "publish" for item in effects) + and all(item["state"] == "complete" and item["observed_json"] is not None + for item in effects)) + require_observed_predicates('repair_continuation', 'repair_admission', { + 'row': dict(row), 'current': dict(current) if current else None, 'claim': claim, + 'attempts': [dict(item) for item in attempts], + 'effects': [dict(item) for item in effects], 'grant_present': grant is not None, + }, [ + {'code': 'repair_checkpoint_changed', 'satisfied': checkpoint, + 'missing_or_unknown_fact': 'unchanged stopped run and closed checkpoint', + 'next_action': 'inspect the retained run and closed predecessor'}, + {'code': 'repair_grant_consumed', 'satisfied': grant is None, + 'missing_or_unknown_fact': 'unused original repair grant', + 'next_action': 'inspect the original repair admission'}, + {'code': 'repair_owner_conflict', 'satisfied': owner, + 'missing_or_unknown_fact': 'original or released work claim', + 'next_action': 'inspect the current work owner'}, + {'code': 'repair_attempt_custody_unknown', + 'satisfied': True if attempt_custody else None, + 'missing_or_unknown_fact': 'finished matching attempts and confirmed cleanup', + 'next_action': 'reconcile the original attempt custody'}, + {'code': 'repair_effects_unsettled', + 'satisfied': True if effect_custody else None, + 'missing_or_unknown_fact': 'settled original publication and other effects', + 'next_action': 'reconcile the original effects'}, + ], lifecycle='stopped', owner_proof=( + True if (checkpoint and grant is None and owner and attempt_custody + and effect_custody) else None), + legacy_message=('repair preflight claim authority changed' + if preflight and not owner + else 'repair continuation lost its frozen run or ownership')) + try: work_binding(self, spec, db) - claim = self.state.claim_for(db, spec["work_id"]) - if (claim is not None if stopped_claim else - claim is None or claim["owner"] != f"external:devflow:{run_id}"): - raise ValueError("repair preflight claim authority changed") + except ValueError as exc: + from .delivery_lifecycle import RecoveryRefused + + if isinstance(exc, RecoveryRefused): + raise + require_observed_predicates('repair_continuation', 'repair_admission', { + 'row': dict(row), 'current': dict(current), 'claim': claim, + }, [{ + 'code': 'repair_work_binding_unproven', 'satisfied': None, + 'missing_or_unknown_fact': 'original work, issue and repository binding', + 'next_action': 'inspect the original work and repository authority', + }], lifecycle='stopped', owner_proof=None, legacy_message=str(exc)) + return current + + if preflight: + with self._connect() as db: + db.execute("BEGIN") + observe_admission(db) return {**response, "preflight": True, "diagnostics": findings, "lifecycle_assessment": assessment, "title_constraint_sha256": None} if finalized: @@ -2141,66 +2289,14 @@ def continue_repair( with self._connect() as db: db.execute("BEGIN IMMEDIATE") prior = db.execute( - "SELECT request_digest,response_json FROM delivery_commands WHERE command_id=?", - (command_id,), + "SELECT run_id,request_digest,response_json FROM delivery_commands " + "WHERE command_id=?", (command_id,), ).fetchone() if prior: - if prior["request_digest"] != command_digest: - raise ValueError("command ID already belongs to different inputs") + command_receipt('repair_continuation', prior, run_id, command_digest, + request=supplied, db=db) return json.loads(prior["response_json"]) - current = db.execute("SELECT * FROM delivery_runs WHERE run_id=?", (run_id,)).fetchone() - claim = self.state.claim_for(db, spec["work_id"]) - attempts = db.execute( - "SELECT role,iteration,state,session_id,cleanup " - "FROM delivery_attempts WHERE run_id=?", - (run_id,), - ).fetchall() - effects = db.execute( - "SELECT kind,state,observed_json FROM delivery_effects WHERE run_id=?", - (run_id,), - ).fetchall() - if ( - current is None - or current["request_json"] != row["request_json"] - or current["workflow_id"] != row["workflow_id"] - or current["recovery_json"] != row["recovery_json"] - or current["phase"] != "blocked" - or current["outcome"] != "blocked" - or current["execution_state"] != "blocked" - or current["cleanup"] not in ( - {"none", "confirmed"} if stopped_claim else {"none"} - ) - or current["error"] != state["error"] - or current["protocol_revision"] != state["revision"] - or current["iteration"] != iteration - or json.loads(current["candidate_json"] or "null") != candidate - or json.loads(current["pr_json"] or "null") != pr - or json.loads(current["checks_json"] or "{}") != checks - or db.execute( - "SELECT 1 FROM delivery_repair_grants WHERE run_id=?", (run_id,) - ).fetchone() - or (claim is not None if stopped_claim else - claim is None or claim["owner"] != f"external:devflow:{run_id}") - or len(attempts) != len(roles) - or any( - item["state"] != "finished" or item["cleanup"] != "confirmed" - for item in attempts - ) - or sorted( - (item["role"], item["iteration"], item["session_id"]) - for item in attempts - ) - != sorted( - (item["role"], item["iteration"], item["session_id"]) - for item in roles - ) - or not any(item["kind"] == "publish" for item in effects) - or any( - item["state"] != "complete" or item["observed_json"] is None - for item in effects - ) - ): - raise ValueError("repair continuation lost its frozen run or ownership") + current = observe_admission(db) if stopped_claim: from .delivery_policy_recovery import work_binding diff --git a/runtime/tests/test_delivery_feature_resumed_evidence.py b/runtime/tests/test_delivery_feature_resumed_evidence.py index 973d6064..ce83692d 100644 --- a/runtime/tests/test_delivery_feature_resumed_evidence.py +++ b/runtime/tests/test_delivery_feature_resumed_evidence.py @@ -221,14 +221,31 @@ def test_stale_older_resume_link_blocks_only_implementations_that_ran_under_it( role = codex_receipt(store, child, source=source, output=output) cancelled(store, monkeypatch, parent, child, role, output) row = {"error": None, "checks_json": "{}"} + original_admit = resume_mod.admit + + def observed_admission(owner, run_id, command, **kwargs): + preview = original_admit(owner, run_id, command, preflight=True, **kwargs) + assessment = preview["lifecycle_assessment"] + assert assessment["eligible"] and assessment["normalized_lifecycle"] == "terminal" + assert assessment["transition"]["from"] == ["stopped", "terminal"] + return original_admit(owner, run_id, command, **kwargs) + + monkeypatch.setattr(resume_mod, "admit", observed_admission) first = activities.resume_worker(store, parent, child, row) with store._connect() as db: older = json.loads(db.execute("SELECT recovery_json FROM delivery_runs WHERE run_id=?", (child["run_id"],)).fetchone()[0]) + first_receipt = json.loads(db.execute( + "SELECT response_json FROM delivery_commands WHERE command_id=?", + (older["command"]["command_id"],)).fetchone()[0]) # The resumed execution is cancelled again; a second admission chains onto the first. cancelled(store, monkeypatch, parent, child, role, output, workflow_id=first["workflow_id"], execution="closed-resume", recovery_digest=digest(older), revision=9, finish=False) second = activities.resume_worker(store, parent, child, row) + # The first command still identifies its original admission after the later resume. + preview = original_admit(store, child["run_id"], older["command"], preflight=True) + assert preview["lifecycle_assessment"]["disposition"] == "already_admitted" + assert original_admit(store, child["run_id"], older["command"]) == first_receipt specs = {"current": second["spec"], "middle": first["spec"], "oldest": child} assert len({digest(value) for value in specs.values()}) == 3 if ran_under != "oldest": diff --git a/runtime/tests/test_delivery_gate_retry.py b/runtime/tests/test_delivery_gate_retry.py index cafceb31..682e0a15 100644 --- a/runtime/tests/test_delivery_gate_retry.py +++ b/runtime/tests/test_delivery_gate_retry.py @@ -180,7 +180,7 @@ def first_published_after_repair( (canonical_json(state['checks']),)) grant = {k: v for k, v in request.items() if k != 'continuation_kind'} grant.update(command_id='one-source-grant', additional_iterations=1) - store.continue_repair('run-1', grant) + original_receipt = store.continue_repair('run-1', grant) with store._connect() as db: prior = json.loads(db.execute( "SELECT recovery_json FROM delivery_runs WHERE run_id='run-1'").fetchone()[0]) @@ -198,6 +198,8 @@ def first_published_after_repair( request.update(command_id='post-repair-assessment', expected_iteration=2, expected_revision=14) before_grant = canonical_json(prior) store.continue_repair('run-1', request) + # An earlier authentic command remains replayable after the workflow advances. + assert store.continue_repair('run-1', grant) == original_receipt effective = store.effective_spec('run-1') assert effective['gate_retry_stage'] == 'published' assert effective['policy']['max_repairs'] == store.spec('run-1')['policy']['max_repairs'] @@ -211,6 +213,44 @@ def first_published_after_repair( return store, broker, state, closed, request, current +@pytest.mark.asyncio +async def test_public_historical_repair_replay_after_gate_admission(stopped, monkeypatch): + """A later gate workflow must not invalidate the original repair receipt.""" + import httpx + + from devflow_temporal.delivery_api import create_app + + store, _, _, _, _, current = first_published_after_repair(stopped, monkeypatch) + original = current['original_recovery'] + state = original['state'] + request = {'command_id': 'one-source-grant', 'expected_revision': state['revision'], + 'expected_iteration': state['iteration'], + 'expected_candidate_id': original['candidate']['id'], + 'expected_pr_number': state['pull_request']['number'], + 'expected_pr_head': state['pull_request']['head'], + 'additional_iterations': original['additional_iterations']} + with store._connect() as db: + expected = json.loads(db.execute( + 'SELECT response_json FROM delivery_commands WHERE command_id=?', + (request['command_id'],)).fetchone()[0]) + before = tuple(db.execute(f'SELECT count(*) FROM {table}').fetchone()[0] + for table in ('delivery_commands', 'delivery_repair_grants')) + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': 'repair_continuation', 'request': request}) + replay = await client.post('/api/runs/run-1/continue-repair', json=request, + headers={'Origin': host, 'X-Devflow-CSRF': token}) + assert preview.json()['disposition'] == 'already_admitted' + assert replay.status_code == 200 and replay.json() == expected + with store._connect() as db: + assert tuple(db.execute(f'SELECT count(*) FROM {table}').fetchone()[0] + for table in ('delivery_commands', 'delivery_repair_grants')) == before + + @pytest.fixture def missing_report(stopped, monkeypatch): from copy import deepcopy @@ -246,3 +286,133 @@ def missing_report(stopped, monkeypatch): closed.update(workflow_id=workflow.fetchone()[0], recovery_digest=digest(previous)) return store, broker, state, closed, {**request, 'command_id': 'missing-report-reassessment', 'expected_revision': 23}, previous, reference + + +@pytest.mark.asyncio +async def test_public_gate_retry_rechecks_candidate_and_owner(unpublished, monkeypatch): + """A valid preview cannot authorize changed source or a competing owner.""" + import asyncio + import copy + import threading + from contextlib import contextmanager + + import httpx + + from devflow_temporal import delivery_gate_retry + from devflow_temporal.contracts import digest + from devflow_temporal.delivery_api import create_app + + store, broker, state, command = unpublished + def renew(spec, *_args): + renewed = copy.deepcopy(spec) + renewed['policy']['measured_runtime'] = 'renewed' + renewed['policy_digest'] = digest(renewed['policy']) + return renewed + monkeypatch.setattr(delivery_gate_retry, 'prepare_runtime', renew) + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + headers = {'Origin': host, 'X-Devflow-CSRF': token} + async def assess(): + return await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': PREPUBLICATION_KIND, 'request': command}) + assert (await assess()).json()['eligible'] + path = broker.checkout / 'README.md' + original = path.read_bytes() + path.write_text('Changed after preflight\n') + changed = await assess() + response = await client.post('/api/runs/run-1/continue-repair', json=command, + headers=headers) + assert not changed.json()['eligible'] + assert 'gate_candidate_changed' in {b['code'] for b in changed.json()['blockers']} + assert response.status_code == 409 and response.json()['detail'] == changed.json() + path.write_bytes(original) + with store._connect() as db: + store.state.claim_work(db, broker.spec['work_id'], 'external:other', 'fixture') + owned = await assess() + response = await client.post('/api/runs/run-1/continue-repair', json=command, + headers=headers) + assert not owned.json()['eligible'] + assert 'gate_owner_conflict' in {b['code'] for b in owned.json()['blockers']} + assert owned.json()['proof_digest'] is None + assert response.status_code == 409 and response.json()['detail'] == owned.json() + # A diagnostic that observes this stopped subject owns a different route. + from devflow_temporal import delivery_native_diagnostic as diagnostic + + monkeypatch.setattr(diagnostic, '_readonly', lambda value: value) + diagnostic_request = { + 'command_id': 'diagnostic', 'expected_revision': state['revision'], + 'expected_projection_revision': 0, 'expected_iteration': state['iteration'], + 'expected_candidate_id': command['expected_candidate_id'], + 'expected_candidate_head': command['expected_candidate_head'], + 'expected_content_sha256': '0' * 64, 'expected_pr_number': 1, + 'expected_pr_head': '0' * 40, 'check_id': diagnostic.CHECK_ID, + 'expected_check_sha256': diagnostic.CHECK_SHA256, + } + diagnostic_result = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': diagnostic.KIND, 'request': diagnostic_request}) + assert not diagnostic_result.json()['eligible'] + assert diagnostic_result.json()['operation'] == diagnostic.KIND + with store._connect() as db: + assert not db.execute('SELECT 1 FROM delivery_gate_admissions').fetchone() + store.state.release_work(db, broker.spec['work_id'], 'external:other') + assert (await assess()).json()['eligible'] + # Force both requests to observe the missing command before owner locking. + original_lock = delivery_gate_retry._lock + ready = threading.Barrier(2, timeout=10) + + @contextmanager + def overlapping_lock(path): + ready.wait() + with original_lock(path): + yield + + monkeypatch.setattr(delivery_gate_retry, '_lock', overlapping_lock) + accepted, concurrent = await asyncio.gather(*( + client.post('/api/runs/run-1/continue-repair', json=command, headers=headers) + for _ in range(2))) + assert accepted.status_code == concurrent.status_code == 200, ( + accepted.text, concurrent.text) + assert concurrent.json() == accepted.json() + monkeypatch.setattr(delivery_gate_retry, '_lock', original_lock) + replay = await client.post('/api/runs/run-1/continue-repair', json=command, + headers=headers) + assert replay.json() == accepted.json() + replay_preview = (await assess()).json() + assert not replay_preview['eligible'] + assert replay_preview['disposition'] == 'already_admitted' + assert replay_preview['blockers'][0]['code'] == 'already_admitted' + replay_preflight = await client.post('/api/runs/run-1/repair-admission-preflight', + json=command, headers=headers) + assert replay_preflight.json()['workflow_id'] == accepted.json()['workflow_id'] + assert replay_preflight.json()['additional_iterations'] == 0 + changed_command = {**command, 'expected_revision': command['expected_revision'] + 1} + conflict = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': PREPUBLICATION_KIND, 'request': changed_command}) + rejected = await client.post('/api/runs/run-1/continue-repair', json=changed_command, + headers=headers) + assert conflict.json()['blockers'][0]['code'] == 'command_conflict' + assert rejected.status_code == 409 and rejected.json()['detail'] == conflict.json() + with store._connect() as db: + saved = db.execute('SELECT response_json FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone()[0] + receipt = accepted.json() + for damaged in ({}, {**receipt, 'workflow_id': 'unrelated-workflow'}, + {key: receipt[key] for key in ('run_id', 'phase', 'workflow_id')}, + {**receipt, 'additional_iterations': 1}): + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (json.dumps(damaged), command['command_id'])) + corrupt = await assess() + refused = await client.post('/api/runs/run-1/continue-repair', + json=command, headers=headers) + assert 'command_receipt_unknown' in {b['code'] for b in corrupt.json()['blockers']} + assert refused.status_code == 409 and refused.json()['detail'] == corrupt.json() + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (saved, command['command_id'])) + with store._connect() as db: + assert db.execute('SELECT COUNT(*) FROM delivery_gate_admissions').fetchone()[0] == 1 + assert state['error'] == 'prepublication repair limit exhausted' diff --git a/runtime/tests/test_delivery_pending_publication.py b/runtime/tests/test_delivery_pending_publication.py index 6b1d3c5c..c89a75ae 100644 --- a/runtime/tests/test_delivery_pending_publication.py +++ b/runtime/tests/test_delivery_pending_publication.py @@ -120,3 +120,53 @@ async def test_public_pending_assessment_agrees_with_admission(pending): assert refused.json()['blockers'][0]['code'] == 'revision_changed' accepted = await client.post(path + 'recover-publication', json=request, headers=headers) assert accepted.status_code == 200 and not accepted.json()['implementation_authority'] + replay = await client.post(path + 'recover-publication', json=request, headers=headers) + assert replay.status_code == 200 and replay.json() == accepted.json() + # The prior gate retry remains readable after publication starts a new workflow. + with store._connect() as db: + prior_gate = json.loads(db.execute( + 'SELECT recovery_json FROM delivery_gate_admissions WHERE run_id=?', + ('run-1',)).fetchone()[0])['command'] + prior_gate_response = json.loads(db.execute( + 'SELECT response_json FROM delivery_commands WHERE command_id=?', + (prior_gate['command_id'],)).fetchone()[0]) + historical = await client.post(path + 'continue-repair', json=prior_gate, headers=headers) + assert historical.status_code == 200 and historical.json() == prior_gate_response + replay_preview = await client.post(path + 'assess-transition', json={ + 'kind': 'recover_publication', 'request': request}) + conflict = {**request, 'expected_revision': request['expected_revision'] + 1} + conflict_preview = await client.post(path + 'assess-transition', json={ + 'kind': 'recover_publication', 'request': conflict}) + conflict_admission = await client.post(path + 'recover-publication', json=conflict, + headers=headers) + assert conflict_admission.status_code == 409 + assert conflict_admission.json()['detail'] == conflict_preview.json(), ( + conflict_preview.json(), conflict_admission.json()) + assert conflict_preview.json()['blockers'][0]['code'] == 'command_conflict' + assert replay_preview.json()['disposition'] == 'already_admitted' + assert replay_preview.json()['blockers'][0]['code'] == 'already_admitted' + with store._connect() as db: + assert db.execute('SELECT count(*) FROM delivery_commands WHERE command_id=?', + (request['command_id'],)).fetchone()[0] == 1 + saved = db.execute('SELECT response_json FROM delivery_commands WHERE command_id=?', + (request['command_id'],)).fetchone()[0] + receipt = accepted.json() + for corrupt in ({}, {**receipt, 'workflow_id': 'unrelated-workflow'}, + {key: receipt[key] for key in ('run_id', 'phase', 'workflow_id')}, + {**receipt, 'implementation_authority': True}): + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (json.dumps(corrupt), request['command_id'])) + corrupt_preview = await client.post(path + 'assess-transition', json={ + 'kind': 'recover_publication', 'request': request}) + corrupt_admission = await client.post(path + 'recover-publication', json=request, + headers=headers) + assert corrupt_admission.status_code == 409 + assert corrupt_admission.json()['detail'] == corrupt_preview.json() + assert 'command_receipt_unknown' in { + b['code'] for b in corrupt_preview.json()['blockers']} + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (saved, request['command_id'])) + assert (await client.post(path + 'recover-publication', json=request, + headers=headers)).json() == receipt diff --git a/runtime/tests/test_delivery_publication_self_heal.py b/runtime/tests/test_delivery_publication_self_heal.py index 3bdc6bf6..90810fdc 100644 --- a/runtime/tests/test_delivery_publication_self_heal.py +++ b/runtime/tests/test_delivery_publication_self_heal.py @@ -310,6 +310,32 @@ async def terminal_tracker(payload): # PR readback; the publication lease alone is not native cleanup. actor = subprocess.Popen(['/bin/sleep', '30'], cwd=broker.checkout) try: + import httpx + + from devflow_temporal.delivery_api import create_app + + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + transport = httpx.ASGITransport(app=app, client=('127.0.0.1', 10001)) + async with httpx.AsyncClient(transport=transport, base_url=host) as browser: + token = (await browser.get('/api/session')).json()['csrf_token'] + headers = {'Origin': host, 'X-Devflow-CSRF': token} + path = '/api/runs/run-1/' + preflight = await browser.post(path + 'assess-transition', json={ + 'kind': 'recover_publication', 'request': recover}) + refusal = await browser.post(path + 'recover-publication', json=recover, + headers=headers) + assert preflight.status_code == 200 and not preflight.json()['eligible'] + assert any(item['code'] == 'publication_native_custody_unknown' + and item['satisfied'] is None + for item in preflight.json()['blockers']), ( + preflight.json(), refusal.status_code, refusal.json()) + assert refusal.status_code == 409 and refusal.json()['detail'] == preflight.json() + with store._connect() as db: + assert not db.execute( + 'SELECT 1 FROM delivery_commands WHERE command_id=?', + (recover['command_id'],)).fetchone() with pytest.raises(ValueError, match='live root user|unreadable lease'): await asyncio.to_thread(store.recover_publication, 'run-1', recover) finally: diff --git a/runtime/tests/test_delivery_stopped_resume.py b/runtime/tests/test_delivery_stopped_resume.py index fad409b8..9943f1d2 100644 --- a/runtime/tests/test_delivery_stopped_resume.py +++ b/runtime/tests/test_delivery_stopped_resume.py @@ -39,13 +39,20 @@ def project(store, state): outcome='blocked', cleanup='confirmed', error=state['error']) -@pytest.mark.parametrize('drift', ['source', 'zero']) -def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): +@pytest.mark.parametrize('drift', [ + 'source', 'zero', 'cleanup', 'attempt', 'claim', 'unpassed_source', 'receipt', + 'session_conflict', 'unrecorded_session', 'push_origin', 'scope', +]) +@pytest.mark.asyncio +async def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): store, broker, state, command = stopped if drift == 'source': (broker.checkout / 'README.md').write_text('Unsealed post-stop edit\n') elif drift == 'scope': (broker.checkout / 'foreign.md').write_text('Escaped scope\n') + state['candidate'] = broker.candidate() + state['roles'][0]['candidate'] = state['candidate'] + project(store, state) elif drift == 'claim': with store._connect() as db: store.state.claim_work(db, broker.spec['work_id'], 'external:devflow:run-1', @@ -55,6 +62,17 @@ def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): elif drift == 'attempt': with store._connect() as db: db.execute("UPDATE delivery_attempts SET state='running'") + elif drift == 'unpassed_source': + state['roles'][0]['status'] = 'blocked' + (broker.checkout / 'README.md').write_text('Unsealed blocked implementation\n') + elif drift == 'receipt': + state['roles'][0]['status'] = 'blocked' + elif drift == 'session_conflict': + state['roles'].append({**state['roles'][0], 'session_id': 'different-session'}) + elif drift == 'unrecorded_session': + state['roles'][0]['session_id'] = None + elif drift == 'push_origin': + _git(broker.checkout, 'remote', 'set-url', '--push', 'origin', '/tmp/foreign-origin.git') elif drift == 'remote': _git(broker.checkout, 'push', 'origin', 'HEAD:refs/heads/' + broker.spec['branch']) elif drift == 'origin': @@ -73,6 +91,36 @@ def test_public_resume_rejects_unowned_or_unsealed_stop(stopped, drift): assert not db.execute( "SELECT 1 FROM delivery_commands WHERE command_id='resume-1'").fetchone() assert store.state.claim_for(db, broker.spec['work_id']) is None or drift == 'claim' + import httpx + + from devflow_temporal.delivery_api import create_app + + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), + base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + actual = await client.post('/api/runs/run-1/continue-repair', json=command, + headers={'Origin': host, 'X-Devflow-CSRF': token}) + assert preview.status_code == 200 and not preview.json()['eligible'] + code = { + 'source': 'stopped_candidate_changed', 'zero': 'pending_repair_unproven', + 'cleanup': 'stopped_role_cleanup_unknown', + 'attempt': 'stopped_attempt_custody_unknown', + 'claim': 'stopped_owner_conflict', 'unpassed_source': 'stopped_candidate_changed', + 'receipt': 'stopped_implementation_receipt_unproven', + 'session_conflict': 'stopped_session_conflict', + 'unrecorded_session': 'stopped_session_unrecorded', + 'push_origin': 'stopped_repository_changed', 'scope': 'stopped_scope_changed', + }[drift] + assert code in {item['code'] for item in preview.json()['blockers']} + assert actual.status_code == 409 and actual.json()['detail'] == preview.json() + with store._connect() as db: + assert not db.execute( + "SELECT 1 FROM delivery_commands WHERE command_id='resume-1'").fetchone() finally: if drift == 'plan': monkey.undo() diff --git a/runtime/tests/test_delivery_store_lifecycle_240.py b/runtime/tests/test_delivery_store_lifecycle_240.py index 3bc62e63..0977263d 100644 --- a/runtime/tests/test_delivery_store_lifecycle_240.py +++ b/runtime/tests/test_delivery_store_lifecycle_240.py @@ -31,13 +31,15 @@ def test_scope_stale_request_never_mutates(service, monkeypatch, field, value): @pytest.mark.asyncio async def test_default_repair_public_assessment_and_admission_agree(service, monkeypatch): + import json + import httpx from test_delivery_store import _failed_published_repair_fixture from devflow_temporal.delivery_api import create_app store, request = service - _spec, _state, command = _failed_published_repair_fixture(store, request, monkeypatch) + _spec, state, command = _failed_published_repair_fixture(store, request, monkeypatch) app = create_app(store.config.path) app.state.delivery.store = store host = store.config.dashboard_url @@ -56,5 +58,233 @@ async def test_default_repair_public_assessment_and_admission_agree(service, mon actual = await client.post(base + 'continue-repair', json=stale, headers=headers) assert actual.status_code == 409 and actual.json()['detail'] == preview.json() assert preview.json()['blockers'][0]['code'] == 'revision_changed' + with store._connect() as db: + original_owner = db.execute('SELECT owner FROM claims WHERE work_id=?', + (request['work_id'],)).fetchone()[0] + original_repository = db.execute('SELECT repository FROM works WHERE id=?', + (request['work_id'],)).fetchone()[0] + db.execute("UPDATE claims SET owner='external:devflow:another' WHERE work_id=?", + (request['work_id'],)) + owner_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + owner_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert owner_admission.status_code == 409 + assert owner_admission.json()['detail'] == owner_preview.json() + assert 'repair_owner_conflict' in {b['code'] for b in owner_preview.json()['blockers']} + with store._connect() as db: + assert not db.execute('SELECT 1 FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone() + db.execute('UPDATE claims SET owner=? WHERE work_id=?', + (original_owner, request['work_id'])) + db.execute("UPDATE works SET repository='github.com/example/other' WHERE id=?", + (request['work_id'],)) + work_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + work_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert work_admission.status_code == 409 + assert work_admission.json()['detail'] == work_preview.json() + assert 'repair_work_binding_unproven' in { + b['code'] for b in work_preview.json()['blockers']} + with store._connect() as db: + assert not db.execute('SELECT 1 FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone() + db.execute('UPDATE works SET repository=? WHERE id=?', + (original_repository, request['work_id'])) + db.execute("UPDATE delivery_attempts SET state='running' WHERE role='implement'") + attempt_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + attempt_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert not attempt_preview.json()['eligible'], (attempt_preview.json(), + attempt_admission.json()) + assert attempt_admission.status_code == 409 + assert attempt_admission.json()['detail'] == attempt_preview.json() + assert 'repair_attempt_custody_unknown' in { + b['code'] for b in attempt_preview.json()['blockers']} + with store._connect() as db: + assert not db.execute('SELECT 1 FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone() + db.execute("UPDATE delivery_attempts SET state='finished' WHERE role='implement'") + db.execute("UPDATE delivery_effects SET state='unknown' WHERE kind='publish'") + effect_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + effect_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert not effect_preview.json()['eligible'] and effect_admission.status_code == 409 + assert effect_admission.json()['detail'] == effect_preview.json() + assert 'repair_effects_unsettled' in { + b['code'] for b in effect_preview.json()['blockers']} + with store._connect() as db: + db.execute("UPDATE delivery_effects SET state='complete' WHERE kind='publish'") + state['roles'][0]['cleanup'] = 'unknown' + custody_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + custody_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert custody_admission.status_code == 409 + assert custody_admission.json()['detail'] == custody_preview.json(), ( + custody_preview.json(), custody_admission.json()) + assert 'repair_role_custody_unknown' in { + b['code'] for b in custody_preview.json()['blockers']} + state['roles'][0]['cleanup'] = 'confirmed' + state['checks']['local'] = {'state': 'unknown'} + check_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + check_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert check_admission.status_code == 409 + assert check_admission.json()['detail'] == check_preview.json() + assert 'repair_check_outcome_unknown' in { + b['code'] for b in check_preview.json()['blockers']} + del state['checks']['local'] accepted = await client.post(base + 'continue-repair', json=command, headers=headers) assert accepted.status_code == 200 and accepted.json()['authorized_through_iteration'] == 4 + replay = await client.post(base + 'continue-repair', json=command, headers=headers) + assert replay.status_code == 200 and replay.json() == accepted.json() + replay_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + assert replay_preview.json()['disposition'] == 'already_admitted' + conflict = {**command, 'expected_revision': command['expected_revision'] + 1} + conflict_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': conflict}) + conflict_admission = await client.post(base + 'continue-repair', json=conflict, + headers=headers) + assert conflict_admission.status_code == 409 + assert conflict_admission.json()['detail'] == conflict_preview.json() + assert conflict_preview.json()['blockers'][0]['code'] == 'command_conflict' + another = {**command, 'command_id': 'another-repair-grant'} + grant_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': another}) + grant_admission = await client.post(base + 'continue-repair', json=another, headers=headers) + assert grant_admission.status_code == 409 + assert grant_admission.json()['detail'] == grant_preview.json() + assert 'repair_grant_consumed' in {b['code'] for b in grant_preview.json()['blockers']} + with store._connect() as db: + assert db.execute('SELECT count(*) FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone()[0] == 1 + assert not db.execute('SELECT 1 FROM delivery_commands WHERE command_id=?', + (another['command_id'],)).fetchone() + saved = db.execute('SELECT response_json FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone()[0] + receipt = accepted.json() + corruptions = [ + {}, + {**receipt, 'workflow_id': 'unrelated-workflow'}, + {key: receipt[key] for key in ('run_id', 'phase', 'workflow_id')}, + {**receipt, 'authorized_through_iteration': + receipt['authorized_through_iteration'] + 1}, + ] + for corrupt in corruptions: + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (json.dumps(corrupt), command['command_id'])) + corrupt_preview = await client.post(base + 'assess-transition', json={ + 'kind': 'repair_continuation', 'request': command}) + corrupt_admission = await client.post(base + 'continue-repair', json=command, + headers=headers) + assert corrupt_admission.status_code == 409 + assert corrupt_admission.json()['detail'] == corrupt_preview.json() + assert 'command_receipt_unknown' in { + b['code'] for b in corrupt_preview.json()['blockers']} + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (saved, command['command_id'])) + assert (await client.post(base + 'continue-repair', json=command, + headers=headers)).json() == receipt + + +@pytest.mark.asyncio +async def test_preplan_stop_refuses_implementation_resume_across_restart(service, monkeypatch): + """A closed pre-intake preparation failure must not be resumed as implementation.""" + import asyncio + import copy + import json + import socket + import sys + + import httpx + import uvicorn + + from devflow_temporal.delivery_api import create_app + from devflow_temporal.delivery_config import DeliveryConfig + from devflow_temporal.delivery_store import DeliveryStore + + initial, request = service + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + listener.listen(128) + raw = copy.deepcopy(initial.config.raw) + raw['dashboard_url'] = f'http://127.0.0.1:{listener.getsockname()[1]}' + raw['roles']['intake'] = {'model': 'fixture', 'effort': 'low'} + initial.config.path.write_text(json.dumps(raw)) + store = DeliveryStore(DeliveryConfig.load(initial.config.path)) + request = {key: value for key, value in request.items() if key != 'accepted_plan'} + store.submit(request) + store.mark_start('run-1', accepted=True) + error = 'fixture preparation failed before planning' + store.project('run-1', phase='blocked', execution_state='blocked', event_type='blocked', + message=error, outcome='blocked', cleanup='confirmed', error=error, + protocol_revision=3, iteration=0) + with store._connect() as db: + db.execute("UPDATE delivery_runs SET workflow_id='delivery-run-1' WHERE run_id='run-1'") + store.state.release_work(db, request['work_id'], 'external:devflow:run-1') + row = db.execute('SELECT * FROM delivery_runs WHERE run_id=?', ('run-1',)).fetchone() + saved_request = row['request_json'] + spec = store.submitted_spec('run-1') + assert not spec['accepted_plan'] + closed = {'workflow_id': 'delivery-run-1', 'execution_run_id': 'preplan-stop', + 'request_digest': row['request_digest'], 'recovery_digest': None, + 'result': {'run_id': 'run-1', 'phase': 'blocked', 'execution_state': 'blocked', + 'outcome': 'blocked', 'cleanup': 'confirmed', 'error': error, + 'revision': 3, 'iteration': 0, 'candidate': None, + 'pull_request': None, 'checks': {}}} + monkeypatch.setattr(DeliveryStore, '_completed_temporal_result', lambda *_a, **_k: closed) + command = {'continuation_kind': 'stopped_delivery_resume', 'command_id': 'preplan-resume', + 'expected_revision': 3, 'expected_iteration': 0, + 'expected_candidate_id': '0' * 64, 'expected_candidate_head': '0' * 40, + 'additional_iterations': 1} + + # Reopen the durable store to exercise the same boundary after a restart. + restarted = DeliveryStore(DeliveryConfig.load(initial.config.path)) + app = create_app(restarted.config.path) + app.state.delivery.store = restarted + with restarted._connect() as db: + before = list(db.iterdump()) + host = restarted.config.dashboard_url + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': 'stopped_delivery_resume', 'request': command}) + response = await client.post('/api/runs/run-1/continue-repair', json=command, + headers={'Origin': host, 'X-Devflow-CSRF': token}) + assert preview.status_code == 200 and not preview.json()['eligible'] + assert 'accepted_plan_unavailable' in {b['code'] for b in preview.json()['blockers']} + assert response.status_code == 409 and response.json()['detail'] == preview.json() + expected_assessment = preview.json() + # Exercise the real read-only CLI and its loopback client against this owned API. + assessment_file = initial.config.path.parent / 'preplan-assessment.json' + assessment_file.write_text(json.dumps({'kind': 'stopped_delivery_resume', 'request': command})) + server = uvicorn.Server(uvicorn.Config(app, lifespan='off', log_level='error')) + serving = asyncio.create_task(server.serve(sockets=[listener])) + try: + for _ in range(200): + if server.started: + break + await asyncio.sleep(.01) + assert server.started + process = await asyncio.create_subprocess_exec( + sys.executable, '-m', 'devflow_temporal.delivery_control', '--config', + str(initial.config.path), 'assess-transition', '--id', 'run-1', '--request', + str(assessment_file), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + stdout, stderr = await asyncio.wait_for(process.communicate(), 30) + assert process.returncode == 0, stderr.decode() + assert json.loads(stdout) == expected_assessment + finally: + server.should_exit = True + await asyncio.wait_for(serving, 10) + listener.close() + with restarted._connect() as db: + assert list(db.iterdump()) == before + assert db.execute('SELECT request_json FROM delivery_runs').fetchone()[0] == saved_request diff --git a/runtime/tests/test_delivery_unpublished_cleanup.py b/runtime/tests/test_delivery_unpublished_cleanup.py index c7113b1d..12c2ff34 100644 --- a/runtime/tests/test_delivery_unpublished_cleanup.py +++ b/runtime/tests/test_delivery_unpublished_cleanup.py @@ -157,24 +157,99 @@ def command_for(state, candidate, *, unknown=False): 'additional_iterations': 1 if unknown else 0} -def test_public_unpublished_cleanup_then_existing_finite_continuation(closed_unpublished): +@pytest.mark.asyncio +async def test_public_unpublished_cleanup_then_existing_finite_continuation(closed_unpublished): + import httpx + + from devflow_temporal.delivery_api import create_app + store, broker, state, spec, calls, _external, shape = closed_unpublished original = copy.deepcopy(state) before = broker.candidate() + command = command_for(state, before, unknown=shape == 'unknown') + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + headers = {'Origin': host, 'X-Devflow-CSRF': token} + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + refused = await client.post('/api/runs/run-1/continue-repair', + json=command, headers=headers) + assert preview.status_code == 200 and not preview.json()['eligible'] + assert {'stopped_cleanup_unconfirmed', 'stopped_owner_conflict'} <= { + blocker['code'] for blocker in preview.json()['blockers']} + assert refused.status_code == 409 and refused.json()['detail'] == preview.json() + assert state == original and broker.candidate() == before + assert calls == [] assert reconcile_attempt(store, run_id='run-1', role='cleanup', iteration=0)['changed'] is True assert state == original and broker.candidate() == before assert len(calls) == 2 assert reconcile_attempt(store, run_id='run-1', role='cleanup', iteration=0)['changed'] is False assert len(calls) == 2 # No duplicate tracker effect and no successor admission. assert retry.retry_once(store) == [] # The sealed terminal classification remains terminal. - command = command_for(state, before, unknown=shape == 'unknown') preflight = store.repair_admission_preflight('run-1', command) assert preflight['authorized_through_iteration'] == command['additional_iterations'] with store._connect() as db: assert store.state.claim_for(db, spec['work_id']) is None assert db.execute('SELECT COUNT(*) FROM delivery_runs').fetchone()[0] == 1 assert not db.execute('SELECT 1 FROM delivery_repair_grants').fetchone() - admitted = store.continue_repair('run-1', command) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + headers = {'Origin': host, 'X-Devflow-CSRF': token} + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + assert preview.status_code == 200 and preview.json()['eligible'], preview.text + response = await client.post('/api/runs/run-1/continue-repair', + json=command, headers=headers) + assert response.status_code == 200, response.text + replay = await client.post('/api/runs/run-1/continue-repair', json=command, headers=headers) + assert replay.json() == response.json() + replay_preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + assert replay_preview.json()['disposition'] == 'already_admitted' + assert replay_preview.json()['blockers'][0]['code'] == 'already_admitted' + replay_preflight = await client.post('/api/runs/run-1/repair-admission-preflight', + json=command, headers=headers) + assert replay_preflight.json()['workflow_id'] == response.json()['workflow_id'] + assert replay_preflight.json()['authorized_through_iteration'] == ( + response.json()['authorized_through_iteration']) + conflicting = {**command, 'expected_revision': command['expected_revision'] + 1} + preview_conflict = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': conflicting}) + conflict = await client.post('/api/runs/run-1/continue-repair', json=conflicting, + headers=headers) + assert preview_conflict.json()['blockers'][0]['code'] == 'command_conflict' + assert conflict.status_code == 409 and conflict.json()['detail'] == preview_conflict.json() + with store._connect() as db: + saved = db.execute('SELECT response_json FROM delivery_commands WHERE command_id=?', + (command['command_id'],)).fetchone()[0] + receipt = response.json() + for damaged in ({}, {**receipt, 'workflow_id': 'unrelated-workflow'}, + {key: receipt[key] for key in ('run_id', 'phase', 'workflow_id')}, + {**receipt, 'authorized_through_iteration': + receipt['authorized_through_iteration'] + 1}, + {**receipt, 'admission_sha256': 'invalid'}): + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (json.dumps(damaged), command['command_id'])) + corrupt = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': command}) + rejected = await client.post('/api/runs/run-1/continue-repair', json=command, + headers=headers) + assert 'command_receipt_unknown' in {b['code'] for b in corrupt.json()['blockers']} + assert rejected.status_code == 409 and rejected.json()['detail'] == corrupt.json() + with store._connect() as db: + db.execute('UPDATE delivery_commands SET response_json=? WHERE command_id=?', + (saved, command['command_id'])) + admitted = response.json() + # Idempotent replay returns the first admission without another owner or debit. + assert admitted['authorized_through_iteration'] == preflight['authorized_through_iteration'] + with store._connect() as db: + assert store.state.claim_for(db, spec['work_id'])['owner'] == 'external:devflow:run-1' + assert db.execute('SELECT COUNT(*) FROM delivery_runs').fetchone()[0] == 1 + assert not db.execute('SELECT 1 FROM delivery_repair_grants').fetchone() assert store.continue_repair('run-1', command) == admitted with store._connect() as db: recovery = json.loads(db.execute('SELECT recovery_json FROM delivery_runs').fetchone()[0]) diff --git a/runtime/tests/test_delivery_unused_budget_resume.py b/runtime/tests/test_delivery_unused_budget_resume.py index 0ba925b4..a5bf04ec 100644 --- a/runtime/tests/test_delivery_unused_budget_resume.py +++ b/runtime/tests/test_delivery_unused_budget_resume.py @@ -69,12 +69,30 @@ def test_unused_resume_keeps_the_original_failed_session_and_partial_source(fixe @pytest.mark.parametrize('iterations', [2, 3]) -def test_fixed_resume_cannot_extend_the_ceiling_or_prepare_runtime(fixed_stop, monkeypatch, - iterations): +@pytest.mark.asyncio +async def test_fixed_resume_cannot_extend_the_ceiling_or_prepare_runtime(fixed_stop, monkeypatch, + iterations): store, _, _, command = fixed_stop def unexpected(*_args, **_kwargs): pytest.fail('a budget extension reached runtime preparation') monkeypatch.setattr(resume, 'prepare_runtime', unexpected) + import httpx + + from devflow_temporal.delivery_api import create_app + + app = create_app(store.config.path) + app.state.delivery.store = store + host = store.config.dashboard_url + exhausted = {**command, 'additional_iterations': iterations} + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=host) as client: + token = (await client.get('/api/session')).json()['csrf_token'] + preview = await client.post('/api/runs/run-1/assess-transition', json={ + 'kind': resume.KIND, 'request': exhausted}) + response = await client.post('/api/runs/run-1/continue-repair', json=exhausted, + headers={'Origin': host, 'X-Devflow-CSRF': token}) + assert preview.status_code == 200 and not preview.json()['eligible'] + assert preview.json()['blockers'][0]['code'] == 'product_budget_exhausted' + assert response.status_code == 409 and response.json()['detail'] == preview.json() with pytest.raises(ValueError, match='fixed repair budget'): store.continue_repair('run-1', {**command, 'additional_iterations': iterations}) with store._connect() as db: