diff --git a/runtime/README.md b/runtime/README.md index ed7d86c5..2fb78ef7 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -276,7 +276,8 @@ operation ID carrying the same domain command and request returns the original operation; a conflicting command ID is refused. Observations retain separate identities so a fresh preflight can be followed by its distinct acknowledgement. -The service dispatches persisted pending work on restart. A previously running +The service dispatches persisted pending work on restart, except pending resume +operations described below. A previously running operation is unknown once its exclusive owner is gone; it is not launched again. `POST /api/operations/{id}/reconcile` checks supported original domain receipts without creating another command. Temporal updates record the exact @@ -292,6 +293,28 @@ CLI `operation-create --request envelope.json`, `operation --id ID`, running service. The corresponding MCP tools and dashboard operation panel expose the same state; none reissues an uncertain effect under a fresh ID. +### Queued pause controls + +Pause currently applies only to explicitly admitted, untimed ordinary runs before +their workflow has started. `GET /api/runs/{id}/pause-control`, CLI +`pause-control --id RUN`, MCP `get_pause_control`, and the dashboard report the +same scope, persisted observed state and blockers. A `pause_requested` or +`pausing` state is not a completed pause. Running workflows, feature children, +native work and timed runs are not yet supported by this control. + +The CLI `pause --id RUN --request FILE` and MCP `pause_run` accept an object +with caller-chosen stable `operation_id` and `command_id` values and the +`expected_revision` from pause control. `resume` and `resume_run` additionally +require its `epoch`; resume is an explicit +instruction. Both use the durable operation journal and require an already running +service. A resume operation still pending when the service restarts is held; +the user must explicitly resubmit the same operation to dispatch it. Preserve +the complete request after a timeout, inspect the operation and reconcile an +unknown result before attempting another command. A recovered +acknowledgement records the original intent; read pause control again for current +settlement or activation. The direct API and dashboard remain available with the +same command ID, revision and epoch semantics. + Routine CLI `run --id RUN` and MCP `get_run` read the versioned persisted status projection. They do not reconstruct current execution authority. Use CLI `run-detail --id RUN` or MCP `get_run_detail` for explicit full-detail diff --git a/runtime/src/devflow_temporal/delivery_activities.py b/runtime/src/devflow_temporal/delivery_activities.py index 2f37ec2a..7a04bf05 100644 --- a/runtime/src/devflow_temporal/delivery_activities.py +++ b/runtime/src/devflow_temporal/delivery_activities.py @@ -14,6 +14,7 @@ import sys import threading import time +from dataclasses import replace from pathlib import Path from typing import Any @@ -65,12 +66,20 @@ def database_contention(exc: BaseException) -> bool: class _DatabaseContentionInbound(ActivityInboundInterceptor): async def execute_activity(self, input: ExecuteActivityInput) -> Any: + from .delivery_pause import PauseDeferred + from .delivery_pause_context import activate, split_activity_request + try: # Only fresh ordinary histories opt in; legacy and merge workflows # retain their existing protocols. SDK context selects the execution. from . import delivery_execution_authority as authority info = activity.info() - request = input.args[0] if input.args else None + args = list(input.args) + context = None + if len(args) == 1: + args[0], context = split_activity_request(args[0], info) + adjusted = replace(input, args=args) if context is not None else input + request = args[0] if args else None spec = request.get("spec") if isinstance(request, dict) else None if (info.workflow_type == "DevflowDeliveryWorkflow" and isinstance(spec, dict) and spec.get(authority.VERSION) == 1): @@ -81,10 +90,21 @@ async def execute_activity(self, input: ExecuteActivityInput) -> Any: from .delivery_writer_custody import activity_custody async with activity_custody(spec): - return await super().execute_activity(input) + with activate(context): + return await super().execute_activity(adjusted) finally: authority._ACTIVITY_AUTHORITY.reset(token) - return await super().execute_activity(input) + with activate(context): + return await super().execute_activity(adjusted) + except PauseDeferred as exc: + if (not exc.root_id or not exc.operation_key or not exc.step + or type(exc.epoch) is not int): + raise + raise ApplicationError( + str(exc), {"root_id": exc.root_id, "operation_key": exc.operation_key, + "step": exc.step, "epoch": exc.epoch}, + type="PauseDeferred", non_retryable=True, + ) from exc except sqlite3.OperationalError as exc: if not database_contention(exc): raise @@ -1099,6 +1119,7 @@ def _role_result(request, broker, workspace, review_diff, result): @activity.defn(name="delivery_publish") async def delivery_publish(request: dict[str, Any]) -> dict[str, Any]: + from .delivery_pause import PauseDeferred from .delivery_writer_custody import to_thread def execute() -> dict[str, Any]: @@ -1109,6 +1130,9 @@ def execute() -> dict[str, Any]: return publish(store, broker, request) return broker.publish(request["iteration"], request["candidate"]) + except PauseDeferred: + # The owner proved that this exact handoff never launched. + raise except Exception as exc: if (request["spec"].get("publication_readback_version") == 1 and broker.publication_may_have_effect is False): diff --git a/runtime/src/devflow_temporal/delivery_broker.py b/runtime/src/devflow_temporal/delivery_broker.py index 090c2d62..6f433502 100644 --- a/runtime/src/devflow_temporal/delivery_broker.py +++ b/runtime/src/devflow_temporal/delivery_broker.py @@ -28,25 +28,67 @@ def _run(argv: list[str], *, cwd: Path | None = None, timeout: int = 120) -> str: + from .delivery_pause_context import enabled as pause_enabled + from .delivery_publication_execution import current_mutation from .delivery_publication_execution import inherited_descriptors as publication_descriptors from .delivery_writer_custody import inherited_descriptors as writer_descriptors nul_output = argv[0] == 'git' and '-z' in argv - result = subprocess.run( - argv, cwd=cwd, text=not nul_output, capture_output=True, timeout=timeout, check=False, - pass_fds=tuple(dict.fromkeys((*publication_descriptors(), *writer_descriptors()))), - ) - stdout = result.stdout.decode('utf-8', 'surrogateescape') if nul_output else result.stdout - stderr = result.stderr.decode('utf-8', 'surrogateescape') if nul_output else result.stderr - if result.returncode: + pass_fds = tuple(dict.fromkeys((*publication_descriptors(), *writer_descriptors()))) + mutation = current_mutation() + if mutation is None or not pause_enabled(mutation[0].broker.spec): + result = subprocess.run( + argv, cwd=cwd, text=not nul_output, capture_output=True, timeout=timeout, check=False, + pass_fds=pass_fds, + ) + returncode, stdout, stderr = result.returncode, result.stdout, result.stderr + else: + from .delivery_pause_context import publication_handoff + + owner, step = mutation + with publication_handoff(owner, step) as handoff: + try: + process = subprocess.Popen(argv, cwd=cwd, text=not nul_output, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + pass_fds=pass_fds) + except (FileNotFoundError, PermissionError): + handoff.no_process_started() + raise + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired as exc: + process.kill() + stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired(argv, timeout, output=stdout, stderr=stderr) from exc + except BaseException: + process.kill() + process.communicate() + raise + returncode = process.returncode + stdout = stdout.decode('utf-8', 'surrogateescape') if nul_output else stdout + stderr = stderr.decode('utf-8', 'surrogateescape') if nul_output else stderr + if returncode: raise RuntimeError( - f"command failed ({result.returncode}): {argv[0]} {argv[1]}: " + f"command failed ({returncode}): {argv[0]} {argv[1]}: " + (stderr.strip() or stdout.strip())[:500] ) # Preserve exact filenames, including CR/LF and leading/trailing whitespace. return stdout if nul_output else stdout.strip() +def _staged_index_evidence(checkout: Path) -> dict[str, str]: + """Observe supported index semantics without mutating or restaging.""" + from .delivery_checkpoint_index import observe_index + + def query(path, *args): + return _git(path, *args).encode("utf-8", "surrogateescape") + + try: + return observe_index(checkout, query) + except ValueError as exc: + raise ValueError("staged publication index semantics are unproven") from exc + + def _git(path: Path, *args: str) -> str: return _run( [ @@ -1708,6 +1750,12 @@ def publish(self, iteration: int, input_candidate: dict[str, Any]) -> dict[str, return {"state": "pending", "reason": "original_" + exc.operation + "_readback"} def _publish_owned(self, iteration, input_candidate, owner): + from .delivery_pause_context import enabled as pause_enabled + + pause_stage = pause_enabled(self.spec) + if pause_stage: + # Unsupported index flags can cause Git to omit reviewed worktree bytes. + _staged_index_evidence(self.checkout) # Only this invocation's confirmed pre-mutation failure can release uncertainty. # Any retained original effect may belong to an earlier lost completion. if self.spec.get("publication_readback_version") == 1: @@ -1767,7 +1815,27 @@ def _publish_owned(self, iteration, input_candidate, owner): if path in indexed or os.path.lexists(self.checkout / path)) # Already-staged removals are absent from both index and worktree; # preserve them without passing an unmatched pathspec to git add. - if stageable: + stage = owner.mutation("stage") if pause_stage else {} + if stage.get("state") == "complete": + # Path-only diffs ignore untracked files and index flags. Bind every + # logical index entry recorded after staging, even for pure deletions. + if stage.get("evidence") != _staged_index_evidence(self.checkout): + raise ValueError("staged publication candidate changed during pause") + unstaged = set(_git(self.checkout, "diff", "--name-only", "-z").split("\0")) + staged = set(_git( + self.checkout, "diff", "--cached", "--name-only", "-z").split("\0")) + unstaged.discard("") + staged.discard("") + if unstaged or staged != set(changed): + raise ValueError("staged publication candidate changed during pause") + elif pause_stage: + # Even an already-staged deletion needs a recorded index before + # the commit fence, and an uncertain earlier stage cannot be replayed. + owner.mutate("stage", lambda: _git( + self.checkout, *(['--literal-pathspecs', 'add', '--all', '--', *stageable] + if stageable else ['ls-files', '--stage', '-z'])), + observe_completion=lambda: _staged_index_evidence(self.checkout)) + elif stageable: _git(self.checkout, '--literal-pathspecs', 'add', '--all', '--', *stageable) if _git(self.checkout, "diff", "--cached", "--name-only"): owner.mutate("commit", lambda: _git( diff --git a/runtime/src/devflow_temporal/delivery_client.py b/runtime/src/devflow_temporal/delivery_client.py index 2b78f373..82e85452 100644 --- a/runtime/src/devflow_temporal/delivery_client.py +++ b/runtime/src/devflow_temporal/delivery_client.py @@ -83,6 +83,27 @@ def reconcile_operation(self, operation_id: str) -> dict: "POST", "/api/operations/" + quote(operation_id, safe="") + "/reconcile", {} ) + def pause_control(self, run_id: str) -> dict: + """Read the persisted pause scope, state and blockers without starting a service.""" + return self._request("GET", "/api/runs/" + quote(run_id, safe="") + "/pause-control") + + def _pause_operation(self, run_id: str, action: str, payload: dict[str, Any]) -> dict: + keys = {"operation_id", "command_id", "expected_revision"} + if action == "resume": + keys.add("epoch") + if not isinstance(payload, dict) or set(payload) != keys: + raise ValueError("pause operation requires the exact operation and command fields") + return self.create_operation({ + "operation_id": payload["operation_id"], "run_id": run_id, "kind": action, + "request": {key: value for key, value in payload.items() if key != "operation_id"}, + }) + + def pause(self, run_id: str, payload: dict[str, Any]) -> dict: + return self._pause_operation(run_id, "pause", payload) + + def resume(self, run_id: str, payload: dict[str, Any]) -> dict: + return self._pause_operation(run_id, "resume", payload) + def submit(self, payload: dict[str, Any]) -> dict: return self._request("POST", "/api/runs", payload) diff --git a/runtime/src/devflow_temporal/delivery_control.py b/runtime/src/devflow_temporal/delivery_control.py index afd8b431..82f1d1a8 100644 --- a/runtime/src/devflow_temporal/delivery_control.py +++ b/runtime/src/devflow_temporal/delivery_control.py @@ -899,6 +899,9 @@ async def worker( "operation", "operation-reconcile", "run-operations", + "pause-control", + "pause", + "resume", "start", "status", "stop", @@ -1002,6 +1005,19 @@ def _run(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: }[args.command](args.id) print(json.dumps(result, sort_keys=True, indent=2)) return + if args.command in {"pause-control", "pause", "resume"}: + if not args.id: + parser.error("--id is required for pause controls") + caller = read_only_client(Path(args.config).expanduser().resolve(strict=True)) + if args.command == "pause-control": + result = caller.pause_control(args.id) + else: + if args.request is None: + parser.error("--request is required for pause and resume") + payload = json.loads(args.request.read_text(encoding="utf-8")) + result = getattr(caller, args.command)(args.id, payload) + print(json.dumps(result, sort_keys=True, indent=2)) + return if args.command == "capture-retained-closures": if args.request is None: parser.error("--request is required for explicit retained-history capture") diff --git a/runtime/src/devflow_temporal/delivery_mcp.py b/runtime/src/devflow_temporal/delivery_mcp.py index 24a51e22..e85da8b2 100644 --- a/runtime/src/devflow_temporal/delivery_mcp.py +++ b/runtime/src/devflow_temporal/delivery_mcp.py @@ -136,6 +136,39 @@ def read_evidence(run_id: str, evidence_id: str) -> dict: """Read one indexed, contained evidence artifact for a run.""" return read_only_client(config_path).evidence(run_id, evidence_id) + @server.tool(annotations=read) + async def get_pause_control(run_id: str) -> dict: + """Read current pause scope, eligibility, observed state and blockers. + + Only explicitly admitted before-start runs currently support pause. + This read does not start the service or authorize a transition. + """ + return await asyncio.to_thread(read_only_client(config_path).pause_control, run_id) + + @server.tool(annotations=write) + async def pause_run(run_id: str, request_json: str) -> dict: + """Request an eligible queued pause through a durable operation. + + Supply stable operation_id and command_id values and the expected_revision + from pause control. A succeeded result acknowledges pause_requested, not + that work is already paused; a pending operation has not yet done so. Keep + the complete request stable and inspect the operation after uncertainty. + The service must already be running. + """ + return await asyncio.to_thread( + read_only_client(config_path).pause, run_id, json.loads(request_json)) + + @server.tool(annotations=write) + async def resume_run(run_id: str, request_json: str) -> dict: + """Explicitly request resume of an eligible queued pause. + + Supply stable operation_id and command_id values and the expected_revision + and epoch from pause control. A resuming acknowledgement is not completed activation. Keep + the request stable after uncertainty; the service must already run. + """ + return await asyncio.to_thread( + read_only_client(config_path).resume, run_id, json.loads(request_json)) + @server.tool(annotations=write) async def answer_decision(run_id: str, request_json: str) -> dict: """Answer a question, review a plan, or authorize the recorded feature merge.""" diff --git a/runtime/src/devflow_temporal/delivery_operation_receipts.py b/runtime/src/devflow_temporal/delivery_operation_receipts.py index 104851d5..caeee665 100644 --- a/runtime/src/devflow_temporal/delivery_operation_receipts.py +++ b/runtime/src/devflow_temporal/delivery_operation_receipts.py @@ -4,7 +4,7 @@ import json -from .contracts import digest +from .contracts import canonical_json, digest _PHASES = { "recover-publication": {"publication_recovery_queued"}, @@ -20,11 +20,76 @@ } +def settle_pause_refusal(store, operation, timestamp): + """Settle only an exact operation bound to durable no-effect owner evidence.""" + from .delivery_pause import _command_identity, refusal_from_row + + kind, run_id, payload = operation["kind"], operation["run_id"], operation["request"] + if (kind not in {"pause", "resume"} or _command_identity(run_id, kind, payload) is None + or operation.get("domain_command_id") != payload["command_id"] + or operation.get("request_digest") != digest({ + "run_id": run_id, "kind": kind, "request": payload})): + return False + with store._connect() as db: + db.execute("BEGIN IMMEDIATE") + current = db.execute("SELECT * FROM delivery_operations WHERE operation_id=?", + (operation["operation_id"],)).fetchone() + if (current is None or current["state"] not in {"running", "unknown"} + or current["owner_generation"] != operation.get("owner_generation") + or current["kind"] != kind or current["run_id"] != run_id + or current["domain_command_id"] != payload["command_id"] + or current["request_digest"] != operation["request_digest"] + or canonical_json(json.loads(current["request_json"])) != canonical_json(payload)): + return False + row = db.execute("SELECT * FROM delivery_pause_commands WHERE command_id=?", + (payload["command_id"],)).fetchone() + refusal = refusal_from_row(row, run_id, kind, payload) + if refusal is None: + return False + db.execute("""UPDATE delivery_operations SET state='refused',updated_at=?,finished_at=?, + progress='refused',result_json=NULL,error=? WHERE operation_id=?""", + (timestamp, timestamp, refusal, operation["operation_id"])) + return True + + def receipt(store, operation): kind, run_id, payload = operation["kind"], operation["run_id"], operation["request"] command_id = payload.get("command_id") if not isinstance(command_id, str): return None + if kind in {"pause", "resume"}: + # A stored acknowledgement is historical evidence of intent, not a + # fresh observation that settlement or activation has completed. + keys = {"command_id", "expected_revision"} | ({"epoch"} if kind == "resume" else set()) + revision = payload.get("expected_revision") + epoch = payload.get("epoch") + if (set(payload) != keys or not command_id or len(command_id) > 512 + or operation.get("domain_command_id") != command_id + or operation.get("request_digest") != digest({ + "run_id": run_id, "kind": kind, "request": payload}) + or type(revision) is not int or revision < 0 + or (kind == "resume" and (type(epoch) is not int or epoch < 1))): + return None + expected = {"action": kind, "root_id": run_id, "expected_revision": revision} + if kind == "resume": + expected["epoch"] = epoch + with store._connect() as db: + row = db.execute( + "SELECT root_id,request_digest,response_json FROM delivery_pause_commands " + "WHERE command_id=?", (command_id,), + ).fetchone() + if row is None or row["root_id"] != run_id or row["request_digest"] != digest(expected): + return None + response = json.loads(row["response_json"]) + state = "pause_requested" if kind == "pause" else "resuming" + if (not isinstance(response, dict) + or set(response) != {"root_id", "state", "revision", "epoch"} + or response["root_id"] != run_id or response["state"] != state + or type(response["revision"]) is not int or response["revision"] != revision + 1 + or type(response["epoch"]) is not int or response["epoch"] < 1 + or (kind == "resume" and response["epoch"] != epoch)): + return None + return response if kind == "issue-context": if set(payload) != {"command_id"}: return None diff --git a/runtime/src/devflow_temporal/delivery_operations.py b/runtime/src/devflow_temporal/delivery_operations.py index 8bb1f83f..e20b18f9 100644 --- a/runtime/src/devflow_temporal/delivery_operations.py +++ b/runtime/src/devflow_temporal/delivery_operations.py @@ -12,6 +12,7 @@ import json import os import re +import sqlite3 import uuid from datetime import UTC, datetime from pathlib import Path @@ -44,6 +45,8 @@ "amend-scope": ("amend_scope", "admission"), "decision": ("decision", "execution"), "cancel": ("cancel", "execution"), + "pause": ("request_pause", "execution"), + "resume": ("request_resume", "execution"), "disable-limits": ("disable_delivery_limits", "execution"), "reconcile-tracker": ("reconcile_tracker", "execution"), } @@ -65,6 +68,8 @@ def _domain_namespaces(kind): # These are the existing owners' command journals. A preflight is never # bound here, so its ID may be reused by its distinct acknowledgement. + if kind in {"pause", "resume"}: + return {"pause-commands"} if kind == "issue-context": return {"issue-context"} if kind in {"decision", "cancel", "disable-limits"}: @@ -131,6 +136,9 @@ def _row(row) -> dict: "refused": "inspect_error", "failed": "inspect_error", }[value["state"]] + if (value["kind"] == "resume" and value["state"] == "pending" + and value["progress"] == "explicit resume required after restart"): + value["next_action"] = "resubmit_same_operation" return value @@ -349,6 +357,14 @@ async def invoke(): if rejection is not None: _set(service.store, operation_id, "refused", error=rejection) return + if operation["kind"] in {"pause", "resume"} and exc.status_code == 409: + from .delivery_operation_receipts import settle_pause_refusal + + try: + if settle_pause_refusal(service.store, operation, now()): + return + except (ValueError, TypeError, KeyError, OSError, sqlite3.Error): + pass # Missing or invalid owner evidence is still unknown. state = ( "refused" if exc.status_code in (400, 403, 404, 422) @@ -399,14 +415,22 @@ def dispatch(operation_id): async def resume_pending(): with service.store._connect() as db: + # A resume queued but not admitted before restart must not + # open a pause without another explicit submission of the same + # operation. Running/unknown commands still reconcile by receipt. + db.execute( + "UPDATE delivery_operations SET progress=?,updated_at=? " + "WHERE kind='resume' AND state='pending'", + ("explicit resume required after restart", now()), + ) rows = db.execute( - "SELECT operation_id,state FROM delivery_operations " + "SELECT operation_id,state,kind FROM delivery_operations " "WHERE state IN ('pending','running')" ).fetchall() for row in rows: - if row["state"] == "pending": + if row["state"] == "pending" and row["kind"] != "resume": dispatch(row["operation_id"]) - else: + elif row["state"] == "running": await asyncio.to_thread(_recover_abandoned, service.store, row["operation_id"]) service.resume_pending_operations = resume_pending @@ -439,6 +463,13 @@ async def create_operation(request: Request): raise HTTPException(400, "run_id is required except for submit") if KINDS[kind][1] != "observation" or kind == "issue-context": mutationguard(request) + if kind in {"pause", "resume"}: + from .delivery_pause import _command_identity + + if _command_identity(run_id, kind, payload) is None: + raise HTTPException( + 400, "pause operation requires exact command identity and fields" + ) request_digest = digest({"run_id": run_id, "kind": kind, "request": payload}) command_id = payload.get("command_id") if KINDS[kind][1] == "observation" or not isinstance(command_id, str) or not command_id: @@ -547,6 +578,14 @@ async def reconcile_operation(request: Request, operation_id: str): return operation if operation["state"] != "unknown": return operation + if operation["kind"] in {"pause", "resume"}: + from .delivery_operation_receipts import settle_pause_refusal + + try: + if await asyncio.to_thread(settle_pause_refusal, service.store, operation, now()): + return await asyncio.to_thread(read, service.store, operation_id) + except (ValueError, TypeError, KeyError, OSError, sqlite3.Error): + pass # A changed or unavailable owner receipt cannot prove refusal. payload, run_id, kind = ( operation["request"], operation["run_id"], diff --git a/runtime/src/devflow_temporal/delivery_pause.py b/runtime/src/devflow_temporal/delivery_pause.py index 03c99bb1..4f9b62db 100644 --- a/runtime/src/devflow_temporal/delivery_pause.py +++ b/runtime/src/devflow_temporal/delivery_pause.py @@ -48,6 +48,11 @@ def initialize(db): epoch INTEGER NOT NULL, workflow_id TEXT NOT NULL, execution_run_id TEXT NOT NULL, owner_generation INTEGER NOT NULL, admitted_at TEXT NOT NULL, PRIMARY KEY(root_id, operation_key, step))""") + db.execute("""CREATE TABLE IF NOT EXISTS delivery_pause_denials ( + root_id TEXT NOT NULL, operation_key TEXT NOT NULL, step TEXT NOT NULL, + epoch INTEGER NOT NULL, workflow_id TEXT NOT NULL, execution_run_id TEXT NOT NULL, + owner_generation INTEGER NOT NULL, denied_at TEXT NOT NULL, + PRIMARY KEY(root_id, operation_key, step, epoch))""") db.execute("""CREATE TABLE IF NOT EXISTS delivery_pause_pending_merges ( root_id TEXT NOT NULL, workflow_id TEXT NOT NULL, grant_sha256 TEXT NOT NULL, state TEXT NOT NULL, execution_run_id TEXT, PRIMARY KEY(root_id, workflow_id))""") @@ -65,6 +70,111 @@ def initialize(db): initialize_intake_pump(db) +_REFUSALS = { + "scope": "pause control scope is not enabled by this service", + "epoch": "invalid pause epoch", + "revision": "invalid control revision", + "conflict": "pause command ID reused with different inputs", + "pause_state": "pause control state or revision changed", + "start_admitted": "intake pause is unavailable after start admission", + "resume_state": "resume control state, revision or epoch changed", + "checkpoint": "checkpoint requires external effect reconciliation before resume", + "clock": "pause workflow clock start acknowledgements are pending", +} + + +class PauseRefused(ValueError): + """A named rejection established before this pause command changed state.""" + + def __init__(self, reason): + self.reason = reason + super().__init__(_REFUSALS[reason]) + + +def _command_digest(root_id, action, body): + request = {"action": action, "root_id": root_id, + "expected_revision": body["expected_revision"]} + if action == "resume": + request["epoch"] = body["epoch"] + return digest(request) + + +def _command_identity(root_id, action, body): + keys = {"command_id", "expected_revision"} | ({"epoch"} if action == "resume" else set()) + if (action not in {"pause", "resume"} or not isinstance(body, dict) or set(body) != keys + or not isinstance(root_id, str) or not root_id or len(root_id) > 512 + or not isinstance(body["command_id"], str) or not body["command_id"] + or len(body["command_id"]) > 512): + return None + return _command_digest(root_id, action, body) + + +def _authenticated_command(row): + """Check the identity available in a historical success or refusal.""" + response = json.loads(row["response_json"]) + if not isinstance(response, dict): + return False + if response.get("state") == "refused": + action, body = response.get("action"), response.get("command") + return (isinstance(body, dict) and body.get("command_id") == row["command_id"] + and _command_identity(row["root_id"], action, body) == row["request_digest"] + and refusal_from_row(row, row["root_id"], action, body) is not None) + if (set(response) != {"root_id", "state", "revision", "epoch"} + or response["root_id"] != row["root_id"] + or response["state"] not in {"pause_requested", "resuming"} + or type(response["revision"]) is not int or response["revision"] < 1 + or type(response["epoch"]) is not int or response["epoch"] < 0 + or (response["state"] == "pause_requested" and response["epoch"] < 1)): + return False + action = "pause" if response["state"] == "pause_requested" else "resume" + body = {"command_id": row["command_id"], + "expected_revision": response["revision"] - 1} + if action == "resume": + body["epoch"] = response["epoch"] + return _command_identity(row["root_id"], action, body) == row["request_digest"] + + +def refusal_from_row(row, root_id, action, body): + """Read a bound refusal, or the immutable conflicting command occupying its ID.""" + expected = _command_identity(root_id, action, body) + if expected is None or row is None: + return None + if row["request_digest"] != expected: + # An occupied ID is evidence of conflict only if its own retained + # identity authenticates; a damaged digest is not a refusal. + return _REFUSALS["conflict"] if _authenticated_command(row) else None + if row["root_id"] != root_id: + return None + response = json.loads(row["response_json"]) + if (not isinstance(response, dict) + or set(response) != {"state", "root_id", "action", "command", "reason", "digest"} + or response["state"] != "refused" or response["root_id"] != root_id + or response["action"] != action or response["digest"] != expected + or canonical_json(response["command"]) != canonical_json(body) + or response["reason"] not in _REFUSALS): + return None + return _REFUSALS[response["reason"]] + + +def remember_refusal(store, root_id, action, body, reason): + """Reserve an exactly identified no-effect command before returning its refusal.""" + expected = _command_identity(root_id, action, body) + if expected is None or reason not in _REFUSALS: + return None + with root_mutex(store, root_id), store._connect() as db: + db.execute("BEGIN IMMEDIATE") + row = db.execute("SELECT * FROM delivery_pause_commands WHERE command_id=?", + (body["command_id"],)).fetchone() + if row is not None: + # A concurrent successful replay must remain a success; never replace it. + return refusal_from_row(row, root_id, action, body) + response = {"state": "refused", "root_id": root_id, "action": action, + "command": body, "reason": reason, "digest": expected} + db.execute("INSERT INTO delivery_pause_commands VALUES (?, ?, ?, ?)", + (body["command_id"], root_id, expected, canonical_json(response))) + return _REFUSALS[reason] + + class PauseDeferred(RuntimeError): """The named step did not start because the owner's pause fence is closed.""" @@ -139,7 +249,7 @@ def request_pause(store, root_id, command_id, expected_revision, *, intake_only= raise ValueError("running-workflow pause is not supported") _nonempty(command_id, "command ID") if type(expected_revision) is not int or expected_revision < 0: - raise ValueError("invalid control revision") + raise PauseRefused("revision") request_digest = digest({"action": "pause", "root_id": root_id, "expected_revision": expected_revision}) with root_mutex(store, root_id), store._connect() as db: @@ -148,11 +258,19 @@ def request_pause(store, root_id, command_id, expected_revision, *, intake_only= (command_id,)).fetchone() if prior is not None: if prior["request_digest"] != request_digest: - raise ValueError("pause command ID reused with different inputs") - return json.loads(prior["response_json"]) + raise PauseRefused("conflict") + if not _authenticated_command(prior): + raise ValueError("invalid pause command receipt") + response = json.loads(prior["response_json"]) + if response["state"] == "refused": + body = {"command_id": command_id, "expected_revision": expected_revision} + if refusal_from_row(prior, root_id, "pause", body) is None: + raise ValueError("invalid pause refusal receipt") + raise PauseRefused(response["reason"]) + return response row = _row(db, root_id) if row["revision"] != expected_revision or row["state"] != "running": - raise ValueError("pause control state or revision changed") + raise PauseRefused("pause_state") if intake_only: run = db.execute("""SELECT r.execution_state,o.state AS outbox_state FROM delivery_runs r JOIN delivery_outbox o USING(run_id) @@ -164,7 +282,7 @@ def request_pause(store, root_id, command_id, expected_revision, *, intake_only= "AND step='start' LIMIT 1", (root_id,)).fetchone() or db.execute("SELECT 1 FROM delivery_pause_actors WHERE root_id=? LIMIT 1", (root_id,)).fetchone()): - raise ValueError("intake pause is unavailable after start admission") + raise PauseRefused("start_admitted") revision, epoch = row["revision"] + 1, row["epoch"] + 1 db.execute("""UPDATE delivery_pause_control SET state='pause_requested', revision=?, epoch=?, checkpoint_json=NULL, updated_at=? WHERE root_id=?""", @@ -199,7 +317,7 @@ def begin_resume(store, root_id, command_id, expected_revision, epoch): _nonempty(command_id, "command ID") _epoch(epoch) if type(expected_revision) is not int or expected_revision < 0: - raise ValueError("invalid control revision") + raise PauseRefused("revision") request = {"action": "resume", "root_id": root_id, "expected_revision": expected_revision, "epoch": epoch} request_digest = digest(request) @@ -209,17 +327,26 @@ def begin_resume(store, root_id, command_id, expected_revision, epoch): (command_id,)).fetchone() if prior is not None: if prior["request_digest"] != request_digest: - raise ValueError("pause command ID reused with different inputs") - return json.loads(prior["response_json"]) + raise PauseRefused("conflict") + if not _authenticated_command(prior): + raise ValueError("invalid pause command receipt") + response = json.loads(prior["response_json"]) + if response["state"] == "refused": + body = {"command_id": command_id, "expected_revision": expected_revision, + "epoch": epoch} + if refusal_from_row(prior, root_id, "resume", body) is None: + raise ValueError("invalid pause refusal receipt") + raise PauseRefused(response["reason"]) + return response row = _row(db, root_id) if (row["state"] != "paused" or row["revision"] != expected_revision or row["epoch"] != epoch): - raise ValueError("resume control state, revision or epoch changed") + raise PauseRefused("resume_state") checkpoint = json.loads(row["checkpoint_json"]) if not checkpoint["resumable"]: - raise ValueError("checkpoint requires external effect reconciliation before resume") + raise PauseRefused("checkpoint") if _missing_clock(db, root_id, epoch, finished=False): - raise ValueError("pause workflow clock start acknowledgements are pending") + raise PauseRefused("clock") revision = row["revision"] + 1 db.execute("""UPDATE delivery_pause_control SET state='resuming', revision=?, updated_at=? WHERE root_id=?""", (revision, _now(), root_id)) @@ -241,6 +368,118 @@ def _missing_clock(db, root_id, epoch, *, finished): + condition + ") LIMIT 1", (epoch, root_id)).fetchone() is not None +@contextmanager +def launch_admission_graph(store, bindings, operation_key, step, *, workflow_id, + execution_run_id, owner_generation, validate_graph, graph_guard=None): + """Admit one actual handoff under every authenticated ancestor fence. + + The caller owns actor and graph authentication; this internal interface does + not accept a validator from a public request. Locks are retained through the + caller's handoff, but the tracking write transaction is committed first. + An admitted step cannot be repeated without a separate readback proving the + outcome. A denial is recorded before it is returned to the owning workflow. + """ + from contextlib import ExitStack + + if not isinstance(bindings, dict) or not bindings or not callable(validate_graph): + raise ValueError("pause graph bindings and validator are required") + for value, name in ((operation_key, "operation key"), (step, "step"), + (workflow_id, "workflow ID"), (execution_run_id, "execution run ID")): + _nonempty(value, name) + if type(owner_generation) is not int or owner_generation < 1: + raise ValueError("invalid pause actor generation") + for root_id, epoch in bindings.items(): + _nonempty(root_id, "root ID") + _epoch(epoch) + + class Handoff: + active = True + + def no_process_started(self): + """Withdraw only after the immediate Popen reports a pre-exec failure.""" + if not self.active: + raise ValueError("pause handoff no longer holds its admission locks") + with store._connect() as db: + db.execute("BEGIN IMMEDIATE") + validate_graph(db, bindings, workflow_id, execution_run_id, owner_generation) + for root_id, epoch in sorted(bindings.items()): + row = _row(db, root_id) + actor = db.execute("""SELECT a.execution_run_id,a.owner_generation,s.state + FROM delivery_pause_actors a JOIN delivery_pause_actor_status s + ON s.root_id=a.root_id AND s.workflow_id=a.workflow_id + WHERE a.root_id=? AND a.workflow_id=?""", + (root_id, workflow_id)).fetchone() + if (row["state"] != "running" or row["epoch"] != epoch or actor is None + or actor["state"] != "running" + or actor["execution_run_id"] != execution_run_id + or actor["owner_generation"] != owner_generation): + raise ValueError("pause actor changed before no-launch acknowledgement") + removed = db.execute("""DELETE FROM delivery_pause_steps WHERE root_id=? + AND operation_key=? AND step=? AND epoch=? AND workflow_id=? + AND execution_run_id=? AND owner_generation=?""", + (root_id, operation_key, step, epoch, workflow_id, + execution_run_id, owner_generation)).rowcount + if removed != 1: + raise ValueError("pause handoff no-launch identity changed") + self.active = False + + with ExitStack() as stack: + for root_id in sorted(bindings): + stack.enter_context(root_mutex(store, root_id)) + if graph_guard is not None: + stack.enter_context(graph_guard()) + denied = None + with store._connect() as db: + db.execute("BEGIN IMMEDIATE") + validate_graph(db, bindings, workflow_id, execution_run_id, owner_generation) + controls = [] + for root_id in sorted(bindings): + row = _row(db, root_id) + actor = db.execute("""SELECT a.execution_run_id,a.owner_generation,s.state + FROM delivery_pause_actors a JOIN delivery_pause_actor_status s + ON s.root_id=a.root_id AND s.workflow_id=a.workflow_id + WHERE a.root_id=? AND a.workflow_id=?""", (root_id, workflow_id)).fetchone() + if (actor is None or actor["execution_run_id"] != execution_run_id + or actor["owner_generation"] != owner_generation + or actor["state"] not in {"running", "resume_pending"}): + raise ValueError("pause graph actor changed or completed") + controls.append((row, actor["state"])) + for row, actor_state in controls: + if (row["state"] == "running" and actor_state == "running" + and row["epoch"] == bindings[row["root_id"]]): + continue + db.execute("""INSERT OR IGNORE INTO delivery_pause_denials + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", (row["root_id"], operation_key, step, + row["epoch"], workflow_id, execution_run_id, owner_generation, _now())) + existing = db.execute("""SELECT * FROM delivery_pause_denials WHERE root_id=? + AND operation_key=? AND step=? AND epoch=?""", + (row["root_id"], operation_key, step, row["epoch"])).fetchone() + if (existing["workflow_id"] != workflow_id + or existing["execution_run_id"] != execution_run_id + or existing["owner_generation"] != owner_generation): + raise ValueError("pause denial belongs to a different actor") + if denied is None: + denied = row + if denied is None: + for row, _actor_state in controls: + old = db.execute("""SELECT 1 FROM delivery_pause_steps WHERE root_id=? + AND operation_key=? AND step=?""", + (row["root_id"], operation_key, step)).fetchone() + if old is not None: + raise ValueError("pause handoff already admitted or outcome unknown") + db.execute("INSERT INTO delivery_pause_steps VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (row["root_id"], operation_key, step, row["epoch"], workflow_id, + execution_run_id, owner_generation, _now())) + if denied is not None: + raise PauseDeferred("pause fence closed before handoff", root_id=denied["root_id"], + operation_key=operation_key, step=step, epoch=denied["epoch"]) + handoff = Handoff() + try: + yield handoff + finally: + handoff.active = False + + @contextmanager def start_admission(store, root_id, workflow_id, owner_generation, *, epoch, validate, retry_selected=False): diff --git a/runtime/src/devflow_temporal/delivery_pause_api.py b/runtime/src/devflow_temporal/delivery_pause_api.py index 902d4bde..61446f1f 100644 --- a/runtime/src/devflow_temporal/delivery_pause_api.py +++ b/runtime/src/devflow_temporal/delivery_pause_api.py @@ -1,6 +1,7 @@ """Public controls for the explicitly admitted queued-intake pause.""" import asyncio import json +import sqlite3 from fastapi import HTTPException, Request @@ -10,7 +11,7 @@ def _admitted(store, root_id): if not intake_enabled(store.submitted_spec(root_id)): - raise ValueError('pause control scope is not enabled by this service') + raise pause.PauseRefused('scope') def status(store, root_id): @@ -79,8 +80,16 @@ async def command(request, run_id, action): if not isinstance(body, dict) or set(body) != keys: raise ValueError('invalid pause command body') if action == 'resume' and (type(body['epoch']) is not int or body['epoch'] < 1): - raise ValueError('invalid pause epoch') + raise pause.PauseRefused('epoch') return await asyncio.to_thread(apply_command, run_id, action, body) + except pause.PauseRefused as exc: + try: + await asyncio.to_thread(pause.remember_refusal, service.store, run_id, + action, body, exc.reason) + except (ValueError, TypeError, OSError, sqlite3.Error): + # A refusal without durable owner evidence remains ambiguous. + pass + raise HTTPException(409, str(exc)) from exc except (ValueError, TypeError) as exc: raise HTTPException(409, str(exc)) from exc diff --git a/runtime/src/devflow_temporal/delivery_pause_context.py b/runtime/src/devflow_temporal/delivery_pause_context.py index ec27eb49..ddb65268 100644 --- a/runtime/src/devflow_temporal/delivery_pause_context.py +++ b/runtime/src/devflow_temporal/delivery_pause_context.py @@ -1,7 +1,21 @@ -"""Recognize only the queued-before-start pause protocol.""" +"""Recognize admitted pause protocols and carry workflow handoff identity.""" +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass from pathlib import Path +@dataclass(frozen=True) +class PauseContext: + run_id: str + workflow_id: str + execution_run_id: str + epochs: dict[str, int] + + +_CURRENT: ContextVar[PauseContext | None] = ContextVar("delivery_pause_context", default=None) + + def enabled(spec): if "pause_control_version" not in spec: return False @@ -37,3 +51,68 @@ def dispatch_enabled(spec): if enabled(spec): raise ValueError("running-workflow pause is not supported") return intake_enabled(spec) + + +def split_activity_request(value, info): + """Remove the workflow's epoch carrier before stable effect digests are made.""" + if not isinstance(value, dict) or not isinstance(value.get("spec"), dict): + return value, None + spec = value["spec"] + if not enabled(spec) or "_pause_epochs" not in value: + return value, None + epochs = value["_pause_epochs"] + if (not isinstance(epochs, dict) or not epochs + or any(not isinstance(root, str) or not root or type(epoch) is not int or epoch < 0 + for root, epoch in epochs.items()) + or not isinstance(spec.get("run_id"), str) or not spec["run_id"] + or not isinstance(info.workflow_id, str) or not info.workflow_id + or not isinstance(info.workflow_run_id, str) or not info.workflow_run_id): + raise ValueError("invalid pause activity authority") + request = dict(value) + del request["_pause_epochs"] + return request, PauseContext(spec["run_id"], info.workflow_id, + info.workflow_run_id, dict(epochs)) + + +@contextmanager +def activate(context): + token = _CURRENT.set(context) + try: + yield + finally: + _CURRENT.reset(token) + + +def publication_handoff(owner, step): + """Fence an ordinary publication subprocess at its actual launch boundary.""" + from .delivery_execution_authority import activity_authority, require_current + from .delivery_pause import launch_admission_graph + + spec, store = owner.broker.spec, owner.broker.store + binding = _CURRENT.get() + if (not enabled(spec) or binding is None or binding.run_id != spec["run_id"] + or spec.get("feature_delivery") or spec.get("feature_worker") + or set(binding.epochs) != {spec["run_id"]}): + raise ValueError("ordinary publication lacks its complete pause actor binding") + authority = activity_authority() + if (authority["run_id"] != binding.run_id + or authority["workflow_id"] != binding.workflow_id + or authority["execution_run_id"] != binding.execution_run_id): + raise ValueError("publication pause actor differs from its SDK authority") + + def validate(db, epochs, workflow_id, execution_run_id, generation): + if (set(epochs) != {spec["run_id"]} or workflow_id != authority["workflow_id"] + or execution_run_id != authority["execution_run_id"] + or generation != authority["generation"]): + raise ValueError("publication pause graph changed") + require_current(db, spec, authority) + actor = db.execute("""SELECT actor_run_id FROM delivery_pause_actors + WHERE root_id=? AND workflow_id=?""", (spec["run_id"], workflow_id)).fetchone() + if actor is None or actor["actor_run_id"] != spec["run_id"]: + raise ValueError("publication pause actor changed") + + return launch_admission_graph( + store, binding.epochs, owner.key, step, workflow_id=binding.workflow_id, + execution_run_id=binding.execution_run_id, owner_generation=authority["generation"], + validate_graph=validate, + ) diff --git a/runtime/src/devflow_temporal/delivery_publication_execution.py b/runtime/src/devflow_temporal/delivery_publication_execution.py index b255e1c9..acff51f4 100644 --- a/runtime/src/devflow_temporal/delivery_publication_execution.py +++ b/runtime/src/devflow_temporal/delivery_publication_execution.py @@ -117,11 +117,14 @@ def check(self, db=None): raise ValueError("publication executor no longer owns this generation") return row - def state(self, operation): + def mutation(self, operation): row = self.check() - return json.loads(row["mutations_json"]).get(operation, {}).get("state") + return json.loads(row["mutations_json"]).get(operation, {}) + + def state(self, operation): + return self.mutation(operation).get("state") - def _record(self, operation, state): + def _record(self, operation, state, *, evidence=None): with self.broker.store._connect() as db: db.execute("BEGIN IMMEDIATE") row = self.check(db) @@ -129,6 +132,8 @@ def _record(self, operation, state): mutations[operation] = { "state": state, "generation": self.generation, "owner": self.token, } + if evidence is not None: + mutations[operation]["evidence"] = evidence db.execute( "UPDATE delivery_publication_executors SET mutations_json=? WHERE effect_key=?", (canonical_json(mutations), self.key), @@ -145,7 +150,7 @@ def permitted(self, operation): and json.loads(row["mutations_json"]).get(operation, {}).get("state") in {None, "not_started"}) - def mutate(self, operation, function): + def mutate(self, operation, function, *, observe_completion=None): from .delivery_writer_custody import require_writer require_writer(self.broker.spec, store=self.broker.store) @@ -158,12 +163,24 @@ def mutate(self, operation, function): # Popen did not launch. No CLI exit code or diagnostic text grants this. self._record(operation, "not_started") raise - except BaseException: + except BaseException as exc: + from .delivery_pause import PauseDeferred + + if isinstance(exc, PauseDeferred): + # The pause admission rejected the exact handoff before Popen. + self._record(operation, "not_started") + raise self._record(operation, "unknown") raise finally: _MUTATION.reset(token) - self._record(operation, "complete") + try: + evidence = observe_completion() if observe_completion is not None else None + except BaseException: + # The mutation launched; failed completion readback is not a no-launch proof. + self._record(operation, "unknown") + raise + self._record(operation, "complete", evidence=evidence) return result def require(self, operation): diff --git a/runtime/tests/test_delivery_mcp.py b/runtime/tests/test_delivery_mcp.py index 7fbbb127..35b08103 100644 --- a/runtime/tests/test_delivery_mcp.py +++ b/runtime/tests/test_delivery_mcp.py @@ -35,6 +35,9 @@ def client(path, factory): run_operations=lambda run_id: {"operations_for": run_id}, create_operation=lambda request: {"created": request}, reconcile_operation=lambda operation_id: {"reconciled": operation_id}, + pause_control=lambda run_id: {"root_id": run_id, "scope": "before_start"}, + pause=lambda run_id, request: {"root_id": run_id, **request}, + resume=lambda run_id, request: {"root_id": run_id, **request}, ) monkeypatch.setattr("devflow_temporal.delivery_mcp.client", lambda path: client(path, "write")) @@ -43,6 +46,9 @@ def client(path, factory): async with create_connected_server_and_client_session(build_server(config)) as session: tools = {tool.name: tool for tool in (await session.list_tools()).tools} assert set(tools) == { + "get_pause_control", + "pause_run", + "resume_run", "get_service", "start_service", "submit_run", @@ -71,6 +77,7 @@ def client(path, factory): "continue_repair", } for name in ( + "get_pause_control", "get_service", "list_runs", "get_run", @@ -87,6 +94,8 @@ def client(path, factory): assert tools[name].annotations.destructiveHint is False assert tools[name].annotations.openWorldHint is False for name in ( + "pause_run", + "resume_run", "start_service", "create_operation", "reconcile_operation", @@ -162,9 +171,26 @@ def client(path, factory): ): result = await session.call_tool(name, arguments) assert not result.isError and json.loads(result.content[0].text) == expected + for name, arguments, expected in ( + ("get_pause_control", {"run_id": "same-run"}, + {"root_id": "same-run", "scope": "before_start"}), + ("pause_run", {"run_id": "same-run", "request_json": + '{"operation_id":"op-pause","command_id":"pause",' + '"expected_revision":1}'}, + {"root_id": "same-run", "operation_id": "op-pause", "command_id": "pause", + "expected_revision": 1}), + ("resume_run", {"run_id": "same-run", "request_json": + '{"operation_id":"op-resume","command_id":"resume",' + '"expected_revision":3,"epoch":1}'}, + {"root_id": "same-run", "operation_id": "op-resume", "command_id": "resume", + "expected_revision": 3, "epoch": 1}), + ): + result = await session.call_tool(name, arguments) + assert not result.isError and json.loads(result.content[0].text) == expected assert seen == [(factory, config) for factory in ( "read", "write", "write", "write", "read", "write", "read", "read", "read", "read", "read", "read", "read", + "read", "read", "read", )] diff --git a/runtime/tests/test_delivery_pause_intake_public.py b/runtime/tests/test_delivery_pause_intake_public.py index a032f4e0..75902a14 100644 --- a/runtime/tests/test_delivery_pause_intake_public.py +++ b/runtime/tests/test_delivery_pause_intake_public.py @@ -3,6 +3,7 @@ import httpx import pytest from temporalio import workflow +from test_delivery_pause_intake import unstarted as unstarted from test_delivery_store import service as service from devflow_temporal.delivery_api import create_app @@ -52,6 +53,53 @@ async def test_public_intake_pause_refuses_a_previously_admitted_start(service, 'expected_revision': status['revision']}) assert rejected.status_code == 409 assert 'after start admission' in rejected.text + # A legacy direct caller has reserved a different rejected command ID. + legacy = {'command_id': 'late-legacy', 'expected_revision': status['revision']} + assert (await client.post('/api/runs/run-1/pause', headers=headers, + json=legacy)).status_code == 409 + conflict_envelope = {'operation_id': 'late-conflict', 'run_id': 'run-1', 'kind': 'pause', + 'request': {**legacy, 'expected_revision': status['revision'] + 1}} + assert (await client.post('/api/operations', headers=headers, + json=conflict_envelope)).status_code in (200, 202) + envelope = {'operation_id': 'late-op', 'run_id': 'run-1', 'kind': 'pause', + 'request': {'command_id': 'late-pause', + 'expected_revision': status['revision']}} + response = await client.post('/api/operations', headers=headers, json=envelope) + assert response.status_code in (200, 202) + import asyncio + + async with asyncio.timeout(10): + while True: + refused = (await client.get('/api/operations/late-op')).json() + if refused['state'] not in {'pending', 'running'}: + break + await asyncio.sleep(.02) + assert refused['state'] == 'refused' and 'after start admission' in refused['error'] + async with asyncio.timeout(10): + while True: + conflict = (await client.get('/api/operations/late-conflict')).json() + if conflict['state'] not in {'pending', 'running'}: + break + await asyncio.sleep(.02) + assert conflict['state'] == 'refused' and 'different inputs' in conflict['error'] + assert (await client.get('/api/runs/run-1/pause-control')).json()['observed'] == 'running' + # A separate ordinary run has no admitted pause scope at all. + ordinary = {**request, 'run_id': 'run-2', 'work_id': 'work-2', + 'command_id': 'command-2', 'branch': 'feat/fixture-2', + 'issue_url': 'https://github.com/example/fixture/issues/4'} + created = await client.post('/api/runs', headers=headers, json=ordinary) + assert created.status_code == 200, created.text + unsupported = {'operation_id': 'unsupported-op', 'run_id': 'run-2', 'kind': 'pause', + 'request': {'command_id': 'unsupported-cmd', 'expected_revision': 0}} + assert (await client.post('/api/operations', headers=headers, + json=unsupported)).status_code in (200, 202) + async with asyncio.timeout(10): + while True: + scope = (await client.get('/api/operations/unsupported-op')).json() + if scope['state'] not in {'pending', 'running'}: + break + await asyncio.sleep(.02) + assert scope['state'] == 'refused' and 'scope is not enabled' in scope['error'] @pytest.mark.asyncio @@ -227,3 +275,382 @@ async def race_before_start(*args, **kwargs): "WHERE root_id='run-1'").fetchone()[0] == 1 assert (await handle.describe()).run_id == description.run_id await handle.terminate('owned fixture cleanup') + + +@pytest.mark.asyncio +async def test_public_pause_operations_reconcile_lost_acknowledgements(unstarted, monkeypatch): + """A lost response never repeats a fenced command or fabricates its outcome.""" + import asyncio + import json + from contextlib import contextmanager + + from test_delivery_pause_intake import begin_intake_settlement, controlled_no_start + + from devflow_temporal import delivery_pause as pause + from devflow_temporal import delivery_pause_intake as intake + from devflow_temporal import delivery_pause_namespace + + _controlled_cutover(monkeypatch) + store, spec, run = unstarted + root = spec['run_id'] + app = create_app(store.config.path) + origin = app.state.delivery.config.dashboard_url + calls = [] + original_pause = pause.request_pause + + def tracked_pause(*args, **kwargs): + calls.append('pause') + return original_pause(*args, **kwargs) + + monkeypatch.setattr(pause, 'request_pause', tracked_pause) + original_scope = delivery_pause_namespace.intake_namespace_scope + + @contextmanager + def fail_on_scope_exit(config): + with original_scope(config): + yield + raise ValueError('fixture namespace revalidation failed after commit') + + async def operation(client, headers, envelope): + response = await client.post('/api/operations', headers=headers, json=envelope) + assert response.status_code in (200, 202), response.text + async with asyncio.timeout(10): + while True: + result = (await client.get('/api/operations/' + envelope['operation_id'])).json() + if result['state'] not in {'pending', 'running'}: + return result + await asyncio.sleep(.02) + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=origin) as client: + session = await client.get('/api/session') + headers = {'Origin': origin, 'X-Devflow-CSRF': session.json()['csrf_token']} + status = (await client.get(f'/api/runs/{root}/pause-control')).json() + stale = {'operation_id': 'stale-op', 'run_id': root, 'kind': 'pause', + 'request': {'command_id': 'stale-cmd', 'expected_revision': 99}} + refused = await operation(client, headers, stale) + assert refused['state'] == 'refused' and 'revision changed' in refused['error'] + assert (await client.get(f'/api/runs/{root}/pause-control')).json()['observed'] == 'running' + assert (await operation(client, headers, stale))['state'] == 'refused' + alias = await client.post('/api/operations', headers=headers, + json={**stale, 'operation_id': 'stale-alias'}) + assert alias.json()['operation_id'] == 'stale-op' + direct = await client.post(f'/api/runs/{root}/pause', headers=headers, + json=stale['request']) + assert direct.status_code == 409 + conflict = await client.post('/api/operations', headers=headers, + json={**stale, 'operation_id': 'stale-conflict', + 'request': {**stale['request'], 'expected_revision': status['revision']}}) + assert conflict.status_code == 409 + # Simulate loss after the owner saved its no-effect receipt. Changed + # transport identity or a corrupted receipt cannot manufacture a refusal. + with store._connect() as db: + receipt = db.execute("SELECT response_json FROM delivery_pause_commands " + "WHERE command_id='stale-cmd'").fetchone()[0] + db.execute("UPDATE delivery_operations SET state='unknown',request_json=? " + "WHERE operation_id='stale-op'", + (json.dumps({**stale['request'], 'expected_revision': 98}),)) + assert (await client.post('/api/operations/stale-op/reconcile', + headers=headers)).json()['state'] == 'unknown' + with store._connect() as db: + db.execute("UPDATE delivery_operations SET request_json=? " + "WHERE operation_id='stale-op'", (json.dumps(stale['request']),)) + bad_receipt = {**json.loads(receipt), 'root_id': 'substituted'} + db.execute("UPDATE delivery_pause_commands SET response_json=? WHERE command_id=?", + (json.dumps(bad_receipt), 'stale-cmd')) + assert (await client.post('/api/operations/stale-op/reconcile', + headers=headers)).json()['state'] == 'unknown' + assert (await client.post(f'/api/runs/{root}/pause', headers=headers, + json=stale['request'])).status_code == 409 + with store._connect() as db: + db.execute("UPDATE delivery_pause_commands SET response_json=? WHERE command_id=?", + (receipt, 'stale-cmd')) + assert (await client.post('/api/operations/stale-op/reconcile', + headers=headers)).json()['state'] == 'refused' + calls.clear() + envelope = {'operation_id': 'pause-op', 'run_id': root, 'kind': 'pause', + 'request': {'command_id': 'pause-cmd', 'expected_revision': status['revision']}} + # The owner commits, then namespace revalidation fails with HTTP 409. + # This is ambiguous until the exact successful owner receipt is read. + monkeypatch.setattr(delivery_pause_namespace, 'intake_namespace_scope', fail_on_scope_exit) + lost = await operation(client, headers, envelope) + monkeypatch.setattr(delivery_pause_namespace, 'intake_namespace_scope', original_scope) + assert lost['state'] == 'unknown' + assert (await client.get(f'/api/runs/{root}/pause-control')).json()[ + 'observed'] == 'pause_requested' + # Alias resolves to the original unknown operation; changed authority is refused. + alias = await client.post('/api/operations', headers=headers, + json={**envelope, 'operation_id': 'pause-alias'}) + assert alias.json()['operation_id'] == 'pause-op' + conflict = await client.post('/api/operations', headers=headers, + json={**envelope, 'operation_id': 'pause-conflict', + 'request': {**envelope['request'], 'expected_revision': 99}}) + assert conflict.status_code == 409 + assert calls == ['pause'] + + # Recreate the API against the same owned database, then reconcile only the + # exact committed domain receipt. No dispatcher or operation is replayed. + restarted = create_app(store.config.path) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=restarted), + base_url=origin) as client: + session = await client.get('/api/session') + headers = {'Origin': origin, 'X-Devflow-CSRF': session.json()['csrf_token']} + with store._connect() as db: + saved_digest = db.execute("SELECT request_digest FROM delivery_pause_commands " + "WHERE command_id='pause-cmd'").fetchone()[0] + db.execute("UPDATE delivery_pause_commands SET request_digest='corrupt' " + "WHERE command_id='pause-cmd'") + assert (await client.post('/api/operations/pause-op/reconcile', + headers=headers)).json()['state'] == 'unknown' + with store._connect() as db: + db.execute("UPDATE delivery_pause_commands SET request_digest=? " + "WHERE command_id='pause-cmd'", (saved_digest,)) + recovered = (await client.post('/api/operations/pause-op/reconcile', + headers=headers)).json() + assert recovered['state'] == 'succeeded' + assert recovered['result'] == {'root_id': root, 'state': 'pause_requested', + 'revision': 1, 'epoch': 1} + assert calls == ['pause'] + begin_intake_settlement(store, root) + intake.seal_unstarted(store, root, 1, run['revision'], + prove_unstarted=controlled_no_start(store)) + paused = pause.publish_paused(store, root, 1) + invalid = {'operation_id': 'invalid-resume-op', 'run_id': root, 'kind': 'resume', + 'request': {'command_id': 'invalid-resume-cmd', + 'expected_revision': paused['revision'], 'epoch': 0}} + assert (await operation(client, headers, invalid))['state'] == 'refused' + assert (await client.post(f'/api/runs/{root}/resume', headers=headers, + json=invalid['request'])).status_code == 409 + with store._connect() as db: + db.execute("UPDATE delivery_operations SET state='unknown' " + "WHERE operation_id='invalid-resume-op'") + assert (await client.post('/api/operations/invalid-resume-op/reconcile', + headers=headers)).json()['state'] == 'refused' + assert (await client.get(f'/api/runs/{root}/pause-control')).json()['observed'] == 'paused' + original_resume = pause.begin_resume + + def lost_resume(*args, **kwargs): + calls.append('resume') + original_resume(*args, **kwargs) + raise OSError('fixture lost acknowledgement after commit') + + monkeypatch.setattr(pause, 'begin_resume', lost_resume) + resume = {'operation_id': 'resume-op', 'run_id': root, 'kind': 'resume', + 'request': {'command_id': 'resume-cmd', 'expected_revision': paused['revision'], + 'epoch': 1}} + # Simulate a restart after transport persistence but before admission. + # Startup must leave this resume pending until the user repeats it. + from devflow_temporal.contracts import digest + + with store._connect() as db: + db.execute("""INSERT INTO delivery_operations + (operation_id,run_id,kind,stage,request_digest,request_json,state, + created_at,updated_at,progress,domain_command_id) + VALUES (?,?,'resume','execution',?,?,'pending','2026','2026','queued',?)""", + (resume['operation_id'], root, + digest({'run_id': root, 'kind': 'resume', 'request': resume['request']}), + json.dumps(resume['request']), resume['request']['command_id'])) + await restarted.state.delivery.resume_pending_operations() + held = (await client.get('/api/operations/resume-op')).json() + assert held['state'] == 'pending' and held['next_action'] == 'resubmit_same_operation' + assert (await client.get(f'/api/runs/{root}/pause-control')).json()['observed'] == 'paused' + assert calls == ['pause'] + assert (await operation(client, headers, resume))['state'] == 'unknown' + assert (await client.get(f'/api/runs/{root}/pause-control')).json()[ + 'observed'] == 'resuming' + result = (await client.post('/api/operations/resume-op/reconcile', headers=headers)).json() + assert result['state'] == 'succeeded' + assert result['result'] == {'root_id': root, 'state': 'resuming', + 'revision': paused['revision'] + 1, 'epoch': 1} + assert calls == ['pause', 'resume'] + # A mismatched request cannot turn another command's receipt into success. + with store._connect() as db: + db.execute("UPDATE delivery_operations SET state='unknown',result_json=NULL," + "request_json=? WHERE operation_id='resume-op'", + ('{"command_id":"resume-cmd","expected_revision":999,"epoch":1}',)) + mismatch = (await client.post('/api/operations/resume-op/reconcile', + headers=headers)).json() + assert mismatch['state'] == 'unknown' + # Even another genuine receipt cannot be substituted for this operation. + with store._connect() as db: + db.execute("UPDATE delivery_operations SET kind='pause',request_json=?," + "domain_command_id='pause-cmd' WHERE operation_id='resume-op'", + ('{"command_id":"pause-cmd","expected_revision":0}',)) + substituted = (await client.post('/api/operations/resume-op/reconcile', + headers=headers)).json() + assert substituted['state'] == 'unknown' + assert calls == ['pause', 'resume'] + # A successful historical receipt still rejects a conflicting request + # and replays unchanged after state advances. + direct_conflict = await client.post(f'/api/runs/{root}/pause', headers=headers, + json={**envelope['request'], 'expected_revision': status['revision'] + 1}) + assert direct_conflict.status_code == 409 + replay = await client.post(f'/api/runs/{root}/pause', headers=headers, + json=envelope['request']) + assert replay.status_code == 200 and replay.json() == recovered['result'] + current = (await client.get(f'/api/runs/{root}/pause-control')).json() + assert current['observed'] == 'resuming' + + +@pytest.mark.asyncio +async def test_cli_and_mcp_use_the_same_durable_pause_commands(unstarted, monkeypatch, tmp_path): + """Actual CLI and MCP clients share the public API and never start the service.""" + import asyncio + import json + import socket + import subprocess + import sys + + import uvicorn + from mcp.shared.memory import create_connected_server_and_client_session + from test_delivery_pause_intake import begin_intake_settlement, controlled_no_start + + from devflow_temporal import delivery_pause as pause + from devflow_temporal import delivery_pause_intake as intake + from devflow_temporal.contracts import digest + from devflow_temporal.delivery_mcp import build_server + + _controlled_cutover(monkeypatch) + store, spec, run = unstarted + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + listener.listen() + listener.setblocking(False) + port = listener.getsockname()[1] + host = f'http://127.0.0.1:{port}' + config = json.loads(store.config.path.read_text()) + config['dashboard_url'] = host + store.config.path.write_text(json.dumps(config)) + app = create_app(store.config.path) + + async def dormant(): + await asyncio.Event().wait() + + app.state.delivery.dispatch_loop = dormant + app.state.delivery.pause_intake_loop = dormant + server = uvicorn.Server(uvicorn.Config(app, host='127.0.0.1', port=port, log_level='error')) + serving = asyncio.create_task(server.serve(sockets=[listener])) + + async def cli(command, request=None, *, error=None): + argv = [sys.executable, '-m', 'devflow_temporal.delivery_control', + '--config', str(store.config.path), command, '--id', spec['run_id']] + if request is not None: + request_path = tmp_path / (command + '.json') + request_path.write_text(json.dumps(request)) + argv += ['--request', str(request_path)] + result = await asyncio.to_thread(subprocess.run, argv, capture_output=True, text=True, + timeout=15) + if error is not None: + assert result.returncode != 0 and error in result.stderr + return None + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + async def completed(client, op): + async with asyncio.timeout(10): + while True: + result = (await client.get('/api/operations/' + op)).json() + if result['state'] not in {'pending', 'running'}: + return result + await asyncio.sleep(.02) + + try: + async with asyncio.timeout(10): + while not server.started: + await asyncio.sleep(.02) + before = await cli('pause-control') + assert before['scope'] == 'before_start' + malformed = [ + ('pause', {'command_id': '', 'expected_revision': 0}), + ('pause', {'command_id': 17, 'expected_revision': 0}), + ('pause', {'command_id': 'x' * 513, 'expected_revision': 0}), + ('pause', {'expected_revision': 0}), + ('pause', {'command_id': 'missing-revision'}), + ('pause', {'command_id': 'extra-pause', 'expected_revision': 0, 'extra': True}), + ('resume', {'command_id': 'missing-epoch', 'expected_revision': 0}), + ('resume', {'command_id': 'extra-resume', 'expected_revision': 0, 'epoch': 1, + 'extra': True}), + ] + with store._connect() as db: + counts = (db.execute('SELECT COUNT(*) FROM delivery_operations').fetchone()[0], + db.execute('SELECT COUNT(*) FROM delivery_pause_commands').fetchone()[0]) + async with httpx.AsyncClient(base_url=host) as client: + session = await client.get('/api/session') + headers = {'Origin': host, 'X-Devflow-CSRF': session.json()['csrf_token']} + for index, (kind, payload) in enumerate(malformed): + response = await client.post('/api/operations', headers=headers, + json={'operation_id': f'malformed-{index}', 'run_id': spec['run_id'], + 'kind': kind, 'request': payload}) + assert response.status_code == 400, response.text + await cli('pause', {'operation_id': 'malformed-cli', 'command_id': '', + 'expected_revision': 0}, error='service HTTP 400') + with store._connect() as db: + assert counts == (db.execute('SELECT COUNT(*) FROM delivery_operations') + .fetchone()[0], + db.execute('SELECT COUNT(*) FROM delivery_pause_commands') + .fetchone()[0]) + assert (await client.get(f"/api/runs/{spec['run_id']}/pause-control")).json() == before + # Existing unaddressable journal rows stay unknown rather than being + # rewritten or treated as proof of a refusal. + payload = {'command_id': '', 'expected_revision': 0} + with store._connect() as db: + db.execute("""INSERT INTO delivery_operations + (operation_id,run_id,kind,stage,request_digest,request_json,state, + created_at,updated_at,progress) + VALUES ('legacy-malformed',?,'pause','execution',?,?,'unknown', + '2026','2026','reconciliation required')""", + (spec['run_id'], digest({'run_id': spec['run_id'], 'kind': 'pause', + 'request': payload}), json.dumps(payload))) + legacy = (await client.post('/api/operations/legacy-malformed/reconcile', + headers=headers)).json() + assert legacy['state'] == 'unknown' + stale = {'operation_id': 'cross-stale', 'command_id': 'cross-stale-cmd', + 'expected_revision': 99} + assert (await cli('pause', stale))['operation_id'] == 'cross-stale' + async with httpx.AsyncClient(base_url=host) as client: + assert (await completed(client, 'cross-stale'))['state'] == 'refused' + async with create_connected_server_and_client_session( + build_server(store.config.path) + ) as mcp: + repeated = await mcp.call_tool('pause_run', { + 'run_id': spec['run_id'], 'request_json': json.dumps(stale)}) + assert not repeated.isError + assert json.loads(repeated.content[0].text)['state'] == 'refused' + request = {'operation_id': 'cross-pause', 'command_id': 'cross-pause-cmd', + 'expected_revision': 0} + first = await cli('pause', request) + assert first['operation_id'] == 'cross-pause' + async with httpx.AsyncClient(base_url=host) as client: + result = await completed(client, 'cross-pause') + assert result['state'] == 'succeeded' and result['result']['state'] == 'pause_requested' + async with create_connected_server_and_client_session( + build_server(store.config.path) + ) as mcp: + status = await mcp.call_tool('get_pause_control', {'run_id': spec['run_id']}) + assert not status.isError + assert json.loads(status.content[0].text)['observed'] == 'pause_requested' + repeated = await mcp.call_tool('pause_run', { + 'run_id': spec['run_id'], 'request_json': json.dumps(request)}) + assert not repeated.isError + assert json.loads(repeated.content[0].text)['operation_id'] == 'cross-pause' + begin_intake_settlement(store, spec['run_id']) + intake.seal_unstarted(store, spec['run_id'], 1, run['revision'], + prove_unstarted=controlled_no_start(store)) + paused = pause.publish_paused(store, spec['run_id'], 1) + resume = {'operation_id': 'cross-resume', 'command_id': 'cross-resume-cmd', + 'expected_revision': paused['revision'], 'epoch': 1} + response = await mcp.call_tool('resume_run', { + 'run_id': spec['run_id'], 'request_json': json.dumps(resume)}) + assert not response.isError + assert json.loads(response.content[0].text)['operation_id'] == 'cross-resume' + result = await completed(client, 'cross-resume') + assert result['state'] == 'succeeded' and result['result']['state'] == 'resuming' + repeated = await cli('resume', resume) + assert repeated['operation_id'] == 'cross-resume' + assert (await cli('pause-control'))['observed'] == 'resuming' + with store._connect() as db: + assert db.execute('SELECT COUNT(*) FROM delivery_pause_commands ' + 'WHERE root_id=?', (spec['run_id'],)).fetchone()[0] == 3 + finally: + server.should_exit = True + await asyncio.wait_for(serving, 10) + listener.close() diff --git a/runtime/tests/test_delivery_publication_execution.py b/runtime/tests/test_delivery_publication_execution.py index 3c8b4fff..526acb7e 100644 --- a/runtime/tests/test_delivery_publication_execution.py +++ b/runtime/tests/test_delivery_publication_execution.py @@ -71,6 +71,31 @@ def event(kind): return store, spec, broker, checked, tmp_path, receipt +def running_publication_authority(store, spec): + """Install a controlled SDK actor for the still-disabled running protocol.""" + from devflow_temporal import delivery_execution_authority as execution + from devflow_temporal import delivery_pause_context as context + from devflow_temporal.contracts import canonical_json + + run_id = spec['run_id'] + workflow_id = 'delivery-' + run_id + spec['pause_control_version'] = 1 + with store._connect() as db: + db.execute('UPDATE delivery_runs SET request_json=? WHERE run_id=?', + (canonical_json(spec), run_id)) + reservation = execution.reserve(store, store.pending_starts()[0], spec) + authority = execution.bind(store, reservation, 'fixture-sdk-run') + with store._connect() as db: + db.execute("INSERT INTO delivery_pause_control VALUES " + "(?,'running',0,0,?,'fixture-sdk-run',1,NULL,?)", + (run_id, workflow_id, '2026-10-29T00:00:00+00:00')) + db.execute('INSERT INTO delivery_pause_actors VALUES (?,?,?,1,?)', + (run_id, workflow_id, 'fixture-sdk-run', run_id)) + db.execute("INSERT INTO delivery_pause_actor_status VALUES (?,?,'running')", + (run_id, workflow_id)) + return authority, context.PauseContext(run_id, workflow_id, 'fixture-sdk-run', {run_id: 0}) + + def events(root): path = root / 'events.jsonl' return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] @@ -179,3 +204,383 @@ async def test_child_cli_keeps_publication_lease_when_python_owner_is_killed(pub child.kill() child.wait(timeout=5) await until(root, 'create-end') + + +def test_publication_handoff_preserves_staged_work_across_a_pause(publication, monkeypatch): + """A fenced commit cannot start and a later continuation does not repeat staging.""" + import threading + from contextlib import contextmanager + from types import SimpleNamespace + + from temporalio.exceptions import ApplicationError + from temporalio.worker import ExecuteActivityInput + + from devflow_temporal import delivery_execution_authority as execution + from devflow_temporal import delivery_pause as pause + from devflow_temporal import delivery_pause_context as context + from devflow_temporal.delivery_broker import BrokerReadbackUnavailable + from devflow_temporal.delivery_config import DeliveryConfig + from devflow_temporal.delivery_store import DeliveryStore + + store, spec, broker, candidate, root, _receipt = publication + run_id = spec['run_id'] + workflow_id = 'delivery-' + run_id + # Public running-pause admission remains disabled; SDK history is controlled. + authority, binding = running_publication_authority(store, spec) + original_handoff = context.publication_handoff + reached = [] + entered, accepted = threading.Event(), threading.Event() + + def request_pause(): + entered.set() + with pause.root_mutex(store, run_id), store._connect() as db: + db.execute('BEGIN IMMEDIATE') + db.execute("UPDATE delivery_pause_control SET state='pause_requested'," + "revision=1,epoch=1 WHERE root_id=?", (run_id,)) + accepted.set() + + @contextmanager + def pause_after_stage(owner, step): + requester = None + try: + with original_handoff(owner, step) as handoff: + if step == 'stage': + # The concurrent request must wait until Popen has been handed + # off under the shared root mutex, before it can close the fence. + requester = threading.Thread(target=request_pause) + requester.start() + assert entered.wait(5) + assert not accepted.wait(.1) + yield handoff + reached.append(step) + if requester is not None: + assert accepted.wait(5) + finally: + if requester is not None: + requester.join(5) + assert not requester.is_alive() + + monkeypatch.setattr(context, 'publication_handoff', pause_after_stage) + token = execution._ACTIVITY_AUTHORITY.set(authority) + original_head = _git(broker.checkout, 'rev-parse', 'HEAD') + # Exercise the actual interceptor and activity; SDK selection is controlled, + # while the persisted actor and publication handoff checks remain real. + class InvokeActivity: + async def execute_activity(self, input): + return await input.fn(*input.args) + + info = SimpleNamespace(workflow_type='DevflowDeliveryWorkflow', workflow_id=workflow_id, + workflow_run_id='fixture-sdk-run') + monkeypatch.setattr(delivery_activities.activity, 'info', lambda: info) + + authenticated_calls = [] + + async def authenticated(_store, _spec): + authenticated_calls.append(_spec['run_id']) + return authority + + monkeypatch.setattr(execution, 'authenticate_activity', authenticated) + inbound = delivery_activities._DatabaseContentionInbound(InvokeActivity()) + request = {'spec': spec, 'iteration': 0, 'candidate': candidate, + '_pause_epochs': {run_id: 0}} + input = ExecuteActivityInput(fn=delivery_activities.delivery_publish, args=[request], + executor=None, headers={}) + try: + with pytest.raises(ApplicationError) as denied: + asyncio.run(inbound.execute_activity(input)) + assert authenticated_calls == [run_id] + assert denied.value.type == 'PauseDeferred' and denied.value.non_retryable + assert denied.value.details == ({'root_id': run_id, 'operation_key': 'publish:run-1:0', + 'step': 'commit', 'epoch': 1},) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert _git(broker.checkout, 'diff', '--cached', '--name-only') == 'README.md' + assert events(root) == [] + with store._connect() as db: + mutations = json.loads(db.execute( + 'SELECT mutations_json FROM delivery_publication_executors').fetchone()[0]) + assert mutations['stage']['state'] == 'complete' + assert len(mutations['stage']['evidence']['stage_sha256']) == 64 + assert mutations['commit']['state'] == 'not_started' + assert db.execute("SELECT COUNT(*) FROM delivery_pause_denials " + "WHERE root_id=? AND step='commit'", (run_id,)).fetchone()[0] == 1 + + # Reconstruct the store; a new process cannot treat the denial as a + # completed commit or launch the old epoch while the fence is closed. + reopened = DeliveryStore(DeliveryConfig.load(store.config.path)) + reopened_broker = DeliveryBroker(reopened, spec) + with context.activate(binding), pytest.raises(pause.PauseDeferred): + reopened_broker.publish(0, candidate) + assert events(root) == [] + with pause.root_mutex(reopened, run_id), reopened._connect() as db: + db.execute('BEGIN IMMEDIATE') + # Stand in for the future authenticated checkpoint/resume owner. + db.execute("UPDATE delivery_pause_control SET state='running',revision=2 " + "WHERE root_id=?", (run_id,)) + resumed = context.PauseContext(run_id, workflow_id, 'fixture-sdk-run', {run_id: 1}) + + # A completed staging step is not authority to commit a different + # index, even if the worktree still has the original candidate bytes. + original_blob = _git(broker.checkout, 'rev-parse', ':README.md') + altered = root / 'altered-staged-content' + altered.write_text('Unexpected staged content\n') + altered_blob = _git(broker.checkout, 'hash-object', '-w', str(altered)) + _git(broker.checkout, 'update-index', '--cacheinfo', '100644', altered_blob, 'README.md') + try: + with context.activate(resumed), pytest.raises(ValueError, match='staged publication'): + reopened_broker.publish(0, candidate) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + _git(broker.checkout, 'update-index', '--cacheinfo', + '100644', original_blob, 'README.md') + + # A staged deletion leaves the file untracked, so the ordinary unstaged + # diff is empty even though committing it would drop reviewed content. + _git(broker.checkout, 'update-index', '--force-remove', 'README.md') + try: + assert (broker.checkout / 'README.md').is_file() + assert not _git(broker.checkout, 'diff', '--name-only') + with context.activate(resumed), pytest.raises(ValueError, match='staged publication'): + reopened_broker.publish(0, candidate) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + _git(broker.checkout, 'update-index', '--add', '--cacheinfo', + '100644', original_blob, 'README.md') + + # Assume-unchanged also hides mismatched staged bytes from git diff. + _git(broker.checkout, 'update-index', '--cacheinfo', '100644', altered_blob, 'README.md') + _git(broker.checkout, 'update-index', '--assume-unchanged', 'README.md') + try: + assert not _git(broker.checkout, 'diff', '--name-only') + with context.activate(resumed), pytest.raises(ValueError, match='staged publication'): + reopened_broker.publish(0, candidate) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + _git(broker.checkout, 'update-index', '--no-assume-unchanged', 'README.md') + _git(broker.checkout, 'update-index', '--cacheinfo', + '100644', original_blob, 'README.md') + + real_create = reopened_broker._create_owned_pr + first_create = True + + def create(base): + nonlocal first_create + if first_create: + first_create = False + with monkeypatch.context() as missing: + missing.setenv('PATH', str(root / 'no-executables')) + return real_create(base) + return real_create(base) + + monkeypatch.setattr(reopened_broker, '_create_owned_pr', create) + with context.activate(resumed): + with pytest.raises(FileNotFoundError): + reopened_broker.publish(0, candidate) + with reopened._connect() as db: + assert db.execute("SELECT COUNT(*) FROM delivery_pause_steps " + "WHERE root_id=? AND step='create'", (run_id,)).fetchone()[0] == 0 + # A real launched create with a lost outcome is not a no-launch + # failure. It must be read back before any new create is allowed. + monkeypatch.setenv('PUBLICATION_FAIL', '1') + with pytest.raises(BrokerReadbackUnavailable): + reopened_broker.reconcile_publish(0, candidate, create_missing=True) + monkeypatch.delenv('PUBLICATION_FAIL') + assert reopened_broker.reconcile_publish(0, candidate, create_missing=True) == { + 'state': 'pending', 'reason': 'original_create_readback'} + assert [item['kind'] for item in events(root)].count('create-start') == 1 + head = _git(broker.checkout, 'rev-parse', 'HEAD') + # Controlled remote readback finds the original PR after the lost reply. + (root / 'receipt.json').write_text(json.dumps([dict(_receipt[0], headRefOid=head)])) + result = reopened_broker.reconcile_publish(0, candidate, create_missing=True) + assert result['number'] == 1 + assert reached.count('stage') == 1 + assert [item['kind'] for item in events(root)].count('create-start') == 1 + finally: + execution._ACTIVITY_AUTHORITY.reset(token) + + +def test_unknown_stage_readback_never_relaunches_or_commits(publication, monkeypatch): + """A completed Git add with unavailable index proof stays an unknown handoff.""" + from devflow_temporal import delivery_broker as broker_module + from devflow_temporal import delivery_execution_authority as execution + from devflow_temporal import delivery_pause_context as context + from devflow_temporal.delivery_config import DeliveryConfig + from devflow_temporal.delivery_store import DeliveryStore + + store, spec, broker, candidate, root, _receipt = publication + authority, binding = running_publication_authority(store, spec) + original_head = _git(broker.checkout, 'rev-parse', 'HEAD') + original_observe = broker_module._staged_index_evidence + + observations = [] + + def unavailable(checkout): + observations.append(True) + if len(observations) == 1: + return original_observe(checkout) + raise OSError('controlled index readback failure') + + token = execution._ACTIVITY_AUTHORITY.set(authority) + try: + # Git can otherwise ignore reviewed worktree bytes during the first add. + _git(broker.checkout, 'update-index', '--assume-unchanged', 'README.md') + try: + with context.activate(binding), pytest.raises(ValueError, match='index semantics'): + broker.publish(0, candidate) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + _git(broker.checkout, 'update-index', '--no-assume-unchanged', 'README.md') + monkeypatch.setattr(broker_module, '_staged_index_evidence', unavailable) + with context.activate(binding), pytest.raises(OSError, match='index readback'): + broker.publish(0, candidate) + with store._connect() as db: + mutations = json.loads(db.execute( + 'SELECT mutations_json FROM delivery_publication_executors').fetchone()[0]) + assert mutations['stage']['state'] == 'unknown' + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + monkeypatch.setattr(broker_module, '_staged_index_evidence', original_observe) + reopened = DeliveryStore(DeliveryConfig.load(store.config.path)) + with context.activate(binding): + assert DeliveryBroker(reopened, spec).publish(0, candidate) == { + 'state': 'pending', 'reason': 'original_stage_readback'} + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + execution._ACTIVITY_AUTHORITY.reset(token) + + +def test_already_staged_deletion_is_bound_before_pause(publication, monkeypatch): + """A deletion with no remaining stageable path still has an exact index seal.""" + from contextlib import contextmanager + + from devflow_temporal import delivery_execution_authority as execution + from devflow_temporal import delivery_pause as pause + from devflow_temporal import delivery_pause_context as context + + store, spec, broker, _candidate, root, _receipt = publication + original_blob = _git(broker.checkout, 'rev-parse', 'HEAD:README.md') + (broker.checkout / 'README.md').unlink() + _git(broker.checkout, 'add', '-u', '--', 'README.md') + candidate = broker.candidate() + original_head = _git(broker.checkout, 'rev-parse', 'HEAD') + authority, binding = running_publication_authority(store, spec) + original_handoff = context.publication_handoff + + @contextmanager + def close_after_stage(owner, step): + with original_handoff(owner, step) as handoff: + yield handoff + if step == 'stage': + with pause.root_mutex(store, spec['run_id']), store._connect() as db: + db.execute('BEGIN IMMEDIATE') + db.execute("UPDATE delivery_pause_control SET state='pause_requested'," + "revision=1,epoch=1 WHERE root_id=?", (spec['run_id'],)) + + monkeypatch.setattr(context, 'publication_handoff', close_after_stage) + token = execution._ACTIVITY_AUTHORITY.set(authority) + try: + with context.activate(binding), pytest.raises(pause.PauseDeferred) as denied: + broker.publish(0, candidate) + assert denied.value.step == 'commit' + with store._connect() as db: + mutations = json.loads(db.execute( + 'SELECT mutations_json FROM delivery_publication_executors').fetchone()[0]) + assert len(mutations['stage']['evidence']['stage_sha256']) == 64 + db.execute("UPDATE delivery_pause_control SET state='running',revision=2 " + "WHERE root_id=?", (spec['run_id'],)) + resumed = context.PauseContext(spec['run_id'], binding.workflow_id, + binding.execution_run_id, {spec['run_id']: 1}) + # Reintroducing the old blob would discard the reviewed deletion. + _git(broker.checkout, 'update-index', '--add', '--cacheinfo', + '100644', original_blob, 'README.md') + try: + with context.activate(resumed), pytest.raises(ValueError, match='staged publication'): + broker.publish(0, candidate) + assert _git(broker.checkout, 'rev-parse', 'HEAD') == original_head + assert events(root) == [] + finally: + _git(broker.checkout, 'update-index', '--force-remove', 'README.md') + with context.activate(resumed): + result = broker.publish(0, candidate) + assert result['number'] == 1 + assert not _git(broker.checkout, 'ls-tree', '--name-only', 'HEAD', 'README.md') + assert [item['kind'] for item in events(root)].count('create-start') == 1 + finally: + execution._ACTIVITY_AUTHORITY.reset(token) + + +def test_child_handoff_stays_fenced_when_only_parent_resumes(service, tmp_path): + """A parent resume cannot authorize a child that retains its own pause token.""" + from devflow_temporal import delivery_pause as pause + from devflow_temporal.delivery_config import DeliveryConfig + from devflow_temporal.delivery_store import DeliveryStore + + store, request = service + store.submit(request) + parent, child = request['run_id'], 'fixture-child' + workflow, sdk_run = 'fixture-child-workflow', 'fixture-child-sdk-run' + with store._connect() as db: + for root in (parent, child): + db.execute("INSERT INTO delivery_pause_control VALUES " + "(?,'running',0,0,?,?,1,NULL,?)", + (root, workflow, sdk_run, '2026-10-29T00:00:00+00:00')) + db.execute('INSERT INTO delivery_pause_actors VALUES (?,?,?,1,?)', + (root, workflow, sdk_run, child)) + db.execute("INSERT INTO delivery_pause_actor_status VALUES (?,?,'running')", + (root, workflow)) + db.execute("UPDATE delivery_pause_control SET state='paused',revision=1,epoch=1 " + "WHERE root_id=?", (child,)) + + def validate(db, roots, actor_workflow, actor_run, generation): + assert db.in_transaction + if (set(roots) != {parent, child} or actor_workflow != workflow + or actor_run != sdk_run or generation != 1): + raise ValueError('fixture graph identity changed') + actors = db.execute('SELECT root_id,actor_run_id FROM delivery_pause_actors ' + 'WHERE workflow_id=?', (workflow,)).fetchall() + if {(row['root_id'], row['actor_run_id']) for row in actors} != { + (parent, child), (child, child)}: + raise ValueError('fixture graph membership changed') + + def admit(current, bindings): + return pause.launch_admission_graph( + current, bindings, 'child-work', 'launch', workflow_id=workflow, + execution_run_id=sdk_run, owner_generation=1, validate_graph=validate) + + marker = tmp_path / 'child-launched' + with pytest.raises(pause.PauseDeferred) as denied: + with admit(store, {parent: 0, child: 0}): + pytest.fail('independently paused child was launched') + assert denied.value.root_id == child and not marker.exists() + with store._connect() as db: + db.execute("UPDATE delivery_pause_control SET state='paused',epoch=1 WHERE root_id=?", + (parent,)) + # Parent resume is independent of the child's closed token. + db.execute("UPDATE delivery_pause_control SET state='running' WHERE root_id=?", (parent,)) + reopened = DeliveryStore(DeliveryConfig.load(store.config.path)) + with pytest.raises(pause.PauseDeferred) as denied: + with admit(reopened, {parent: 1, child: 1}): + pytest.fail('parent resume cleared the child pause') + assert denied.value.root_id == child and not marker.exists() + with reopened._connect() as db: + db.execute("UPDATE delivery_pause_control SET state='running' WHERE root_id=?", (child,)) + db.execute("UPDATE delivery_pause_actor_status SET state='resume_pending' " + "WHERE root_id=? AND workflow_id=?", (child, workflow)) + with pytest.raises(pause.PauseDeferred): + with admit(reopened, {parent: 1, child: 1}): + pytest.fail('child launch preceded its resume acknowledgement') + with reopened._connect() as db: + db.execute("UPDATE delivery_pause_actor_status SET state='running' " + "WHERE root_id=? AND workflow_id=?", (child, workflow)) + with admit(reopened, {parent: 1, child: 1}): + process = subprocess.Popen([sys.executable, '-c', + 'from pathlib import Path; import sys; ' + 'Path(sys.argv[1]).write_text("started")', str(marker)]) + assert process.wait(timeout=5) == 0 and marker.read_text() == 'started' + with pytest.raises(ValueError, match='already admitted'): + with admit(reopened, {parent: 1, child: 1}): + pytest.fail('same unknown handoff was launched twice') diff --git a/runtime/tests/test_delivery_readonly.py b/runtime/tests/test_delivery_readonly.py index 84c27653..88380c6b 100644 --- a/runtime/tests/test_delivery_readonly.py +++ b/runtime/tests/test_delivery_readonly.py @@ -12,6 +12,7 @@ READ_TOOLS = [ ('get_service', {}), ('list_runs', {}), ('get_run', {'run_id': 'absent'}), ('get_run_detail', {'run_id': 'absent'}), + ('get_pause_control', {'run_id': 'absent'}), ('get_operation', {'operation_id': 'absent'}), ('get_run_operations', {'run_id': 'absent'}), ('read_evidence', {'run_id': 'absent', 'evidence_id': 'missing'}), @@ -68,12 +69,14 @@ def request(_caller, method, path): 'run': {'run_id': 'run/one', 'protocol_revision': 7}, } caller.detail('run/one') + caller.pause_control('run/one') caller.operation('operation/one') caller.run_operations('run/one') caller.evidence('run/one', 'artifact/one') assert calls == [ ('GET', '/api/service'), ('GET', '/api/runs'), ('GET', '/api/runs/run%2Fone/status'), ('GET', '/api/runs/run%2Fone'), + ('GET', '/api/runs/run%2Fone/pause-control'), ('GET', '/api/operations/operation%2Fone'), ('GET', '/api/runs/run%2Fone/operations'), ('GET', '/api/runs/run%2Fone/evidence/artifact%2Fone'), ]